xref: /linux/drivers/crypto/omap-sham.c (revision 0ea5c948cb64bab5bc7a5516774eb8536f05aa0d)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Cryptographic API.
4  *
5  * Support for OMAP SHA1/MD5 HW acceleration.
6  *
7  * Copyright (c) 2010 Nokia Corporation
8  * Author: Dmitry Kasatkin <dmitry.kasatkin@nokia.com>
9  * Copyright (c) 2011 Texas Instruments Incorporated
10  *
11  * Some ideas are from old omap-sha1-md5.c driver.
12  */
13 
14 #define pr_fmt(fmt) "%s: " fmt, __func__
15 
16 #include <crypto/engine.h>
17 #include <crypto/hmac.h>
18 #include <crypto/internal/hash.h>
19 #include <crypto/scatterwalk.h>
20 #include <crypto/sha1.h>
21 #include <crypto/sha2.h>
22 #include <linux/err.h>
23 #include <linux/device.h>
24 #include <linux/dma-mapping.h>
25 #include <linux/dmaengine.h>
26 #include <linux/init.h>
27 #include <linux/interrupt.h>
28 #include <linux/io.h>
29 #include <linux/irq.h>
30 #include <linux/kernel.h>
31 #include <linux/module.h>
32 #include <linux/of.h>
33 #include <linux/of_address.h>
34 #include <linux/of_irq.h>
35 #include <linux/platform_device.h>
36 #include <linux/pm_runtime.h>
37 #include <linux/scatterlist.h>
38 #include <linux/slab.h>
39 #include <linux/string.h>
40 
41 #define MD5_DIGEST_SIZE			16
42 
43 #define SHA_REG_IDIGEST(dd, x)		((dd)->pdata->idigest_ofs + ((x)*0x04))
44 #define SHA_REG_DIN(dd, x)		((dd)->pdata->din_ofs + ((x) * 0x04))
45 #define SHA_REG_DIGCNT(dd)		((dd)->pdata->digcnt_ofs)
46 
47 #define SHA_REG_ODIGEST(dd, x)		((dd)->pdata->odigest_ofs + (x * 0x04))
48 
49 #define SHA_REG_CTRL			0x18
50 #define SHA_REG_CTRL_LENGTH		(0xFFFFFFFF << 5)
51 #define SHA_REG_CTRL_CLOSE_HASH		(1 << 4)
52 #define SHA_REG_CTRL_ALGO_CONST		(1 << 3)
53 #define SHA_REG_CTRL_ALGO		(1 << 2)
54 #define SHA_REG_CTRL_INPUT_READY	(1 << 1)
55 #define SHA_REG_CTRL_OUTPUT_READY	(1 << 0)
56 
57 #define SHA_REG_REV(dd)			((dd)->pdata->rev_ofs)
58 
59 #define SHA_REG_MASK(dd)		((dd)->pdata->mask_ofs)
60 #define SHA_REG_MASK_DMA_EN		(1 << 3)
61 #define SHA_REG_MASK_IT_EN		(1 << 2)
62 #define SHA_REG_MASK_SOFTRESET		(1 << 1)
63 #define SHA_REG_AUTOIDLE		(1 << 0)
64 
65 #define SHA_REG_SYSSTATUS(dd)		((dd)->pdata->sysstatus_ofs)
66 #define SHA_REG_SYSSTATUS_RESETDONE	(1 << 0)
67 
68 #define SHA_REG_MODE(dd)		((dd)->pdata->mode_ofs)
69 #define SHA_REG_MODE_HMAC_OUTER_HASH	(1 << 7)
70 #define SHA_REG_MODE_HMAC_KEY_PROC	(1 << 5)
71 #define SHA_REG_MODE_CLOSE_HASH		(1 << 4)
72 #define SHA_REG_MODE_ALGO_CONSTANT	(1 << 3)
73 
74 #define SHA_REG_MODE_ALGO_MASK		(7 << 0)
75 #define SHA_REG_MODE_ALGO_MD5_128	(0 << 1)
76 #define SHA_REG_MODE_ALGO_SHA1_160	(1 << 1)
77 #define SHA_REG_MODE_ALGO_SHA2_224	(2 << 1)
78 #define SHA_REG_MODE_ALGO_SHA2_256	(3 << 1)
79 #define SHA_REG_MODE_ALGO_SHA2_384	(1 << 0)
80 #define SHA_REG_MODE_ALGO_SHA2_512	(3 << 0)
81 
82 #define SHA_REG_LENGTH(dd)		((dd)->pdata->length_ofs)
83 
84 #define SHA_REG_IRQSTATUS		0x118
85 #define SHA_REG_IRQSTATUS_CTX_RDY	(1 << 3)
86 #define SHA_REG_IRQSTATUS_PARTHASH_RDY (1 << 2)
87 #define SHA_REG_IRQSTATUS_INPUT_RDY	(1 << 1)
88 #define SHA_REG_IRQSTATUS_OUTPUT_RDY	(1 << 0)
89 
90 #define SHA_REG_IRQENA			0x11C
91 #define SHA_REG_IRQENA_CTX_RDY		(1 << 3)
92 #define SHA_REG_IRQENA_PARTHASH_RDY	(1 << 2)
93 #define SHA_REG_IRQENA_INPUT_RDY	(1 << 1)
94 #define SHA_REG_IRQENA_OUTPUT_RDY	(1 << 0)
95 
96 #define DEFAULT_TIMEOUT_INTERVAL	HZ
97 
98 #define DEFAULT_AUTOSUSPEND_DELAY	1000
99 
100 /* mostly device flags */
101 #define FLAGS_FINAL		1
102 #define FLAGS_DMA_ACTIVE	2
103 #define FLAGS_OUTPUT_READY	3
104 #define FLAGS_CPU		5
105 #define FLAGS_DMA_READY		6
106 #define FLAGS_AUTO_XOR		7
107 #define FLAGS_BE32_SHA1		8
108 #define FLAGS_SGS_COPIED	9
109 #define FLAGS_SGS_ALLOCED	10
110 #define FLAGS_HUGE		11
111 
112 /* context flags */
113 #define FLAGS_FINUP		16
114 
115 #define FLAGS_MODE_SHIFT	18
116 #define FLAGS_MODE_MASK		(SHA_REG_MODE_ALGO_MASK	<< FLAGS_MODE_SHIFT)
117 #define FLAGS_MODE_MD5		(SHA_REG_MODE_ALGO_MD5_128 << FLAGS_MODE_SHIFT)
118 #define FLAGS_MODE_SHA1		(SHA_REG_MODE_ALGO_SHA1_160 << FLAGS_MODE_SHIFT)
119 #define FLAGS_MODE_SHA224	(SHA_REG_MODE_ALGO_SHA2_224 << FLAGS_MODE_SHIFT)
120 #define FLAGS_MODE_SHA256	(SHA_REG_MODE_ALGO_SHA2_256 << FLAGS_MODE_SHIFT)
121 #define FLAGS_MODE_SHA384	(SHA_REG_MODE_ALGO_SHA2_384 << FLAGS_MODE_SHIFT)
122 #define FLAGS_MODE_SHA512	(SHA_REG_MODE_ALGO_SHA2_512 << FLAGS_MODE_SHIFT)
123 
124 #define FLAGS_HMAC		21
125 #define FLAGS_ERROR		22
126 
127 #define OP_UPDATE		1
128 #define OP_FINAL		2
129 
130 #define OMAP_ALIGN_MASK		(sizeof(u32)-1)
131 #define OMAP_ALIGNED		__attribute__((aligned(sizeof(u32))))
132 
133 #define BUFLEN			SHA512_BLOCK_SIZE
134 #define OMAP_SHA_DMA_THRESHOLD	256
135 
136 #define OMAP_SHA_MAX_DMA_LEN	(1024 * 2048)
137 
138 struct omap_sham_dev;
139 
140 struct omap_sham_reqctx {
141 	struct omap_sham_dev	*dd;
142 	unsigned long		flags;
143 	u8			op;
144 
145 	u8			digest[SHA512_DIGEST_SIZE] OMAP_ALIGNED;
146 	size_t			digcnt;
147 	size_t			bufcnt;
148 	size_t			buflen;
149 
150 	/* walk state */
151 	struct scatterlist	*sg;
152 	struct scatterlist	sgl[2];
153 	int			offset;	/* offset in current sg */
154 	int			sg_len;
155 	unsigned int		total;	/* total request */
156 
157 	u8			buffer[] OMAP_ALIGNED;
158 };
159 
160 struct omap_sham_hmac_ctx {
161 	struct crypto_shash	*shash;
162 	u8			ipad[SHA512_BLOCK_SIZE] OMAP_ALIGNED;
163 	u8			opad[SHA512_BLOCK_SIZE] OMAP_ALIGNED;
164 };
165 
166 struct omap_sham_ctx {
167 	unsigned long		flags;
168 
169 	/* fallback stuff */
170 	struct crypto_shash	*fallback;
171 
172 	struct omap_sham_hmac_ctx base[];
173 };
174 
175 #define OMAP_SHAM_QUEUE_LENGTH	10
176 
177 struct omap_sham_algs_info {
178 	struct ahash_engine_alg	*algs_list;
179 	unsigned int		size;
180 	unsigned int		registered;
181 };
182 
183 struct omap_sham_pdata {
184 	struct omap_sham_algs_info	*algs_info;
185 	unsigned int	algs_info_size;
186 	unsigned long	flags;
187 	int		digest_size;
188 
189 	void		(*copy_hash)(struct ahash_request *req, int out);
190 	void		(*write_ctrl)(struct omap_sham_dev *dd, size_t length,
191 				      int final, int dma);
192 	void		(*trigger)(struct omap_sham_dev *dd, size_t length);
193 	int		(*poll_irq)(struct omap_sham_dev *dd);
194 	irqreturn_t	(*intr_hdlr)(int irq, void *dev_id);
195 
196 	u32		odigest_ofs;
197 	u32		idigest_ofs;
198 	u32		din_ofs;
199 	u32		digcnt_ofs;
200 	u32		rev_ofs;
201 	u32		mask_ofs;
202 	u32		sysstatus_ofs;
203 	u32		mode_ofs;
204 	u32		length_ofs;
205 
206 	u32		major_mask;
207 	u32		major_shift;
208 	u32		minor_mask;
209 	u32		minor_shift;
210 };
211 
212 struct omap_sham_dev {
213 	struct list_head	list;
214 	unsigned long		phys_base;
215 	struct device		*dev;
216 	void __iomem		*io_base;
217 	int			irq;
218 	int			err;
219 	struct dma_chan		*dma_lch;
220 	struct tasklet_struct	done_task;
221 	u8			polling_mode;
222 	u8			xmit_buf[BUFLEN] OMAP_ALIGNED;
223 
224 	unsigned long		flags;
225 	int			fallback_sz;
226 	struct crypto_queue	queue;
227 	struct ahash_request	*req;
228 	struct crypto_engine	*engine;
229 
230 	const struct omap_sham_pdata	*pdata;
231 };
232 
233 struct omap_sham_drv {
234 	struct list_head	dev_list;
235 	spinlock_t		lock;
236 	unsigned long		flags;
237 };
238 
239 static struct omap_sham_drv sham = {
240 	.dev_list = LIST_HEAD_INIT(sham.dev_list),
241 	.lock = __SPIN_LOCK_UNLOCKED(sham.lock),
242 };
243 
244 static int omap_sham_enqueue(struct ahash_request *req, unsigned int op);
245 static void omap_sham_finish_req(struct ahash_request *req, int err);
246 
omap_sham_read(struct omap_sham_dev * dd,u32 offset)247 static inline u32 omap_sham_read(struct omap_sham_dev *dd, u32 offset)
248 {
249 	return __raw_readl(dd->io_base + offset);
250 }
251 
omap_sham_write(struct omap_sham_dev * dd,u32 offset,u32 value)252 static inline void omap_sham_write(struct omap_sham_dev *dd,
253 					u32 offset, u32 value)
254 {
255 	__raw_writel(value, dd->io_base + offset);
256 }
257 
omap_sham_write_mask(struct omap_sham_dev * dd,u32 address,u32 value,u32 mask)258 static inline void omap_sham_write_mask(struct omap_sham_dev *dd, u32 address,
259 					u32 value, u32 mask)
260 {
261 	u32 val;
262 
263 	val = omap_sham_read(dd, address);
264 	val &= ~mask;
265 	val |= value;
266 	omap_sham_write(dd, address, val);
267 }
268 
omap_sham_wait(struct omap_sham_dev * dd,u32 offset,u32 bit)269 static inline int omap_sham_wait(struct omap_sham_dev *dd, u32 offset, u32 bit)
270 {
271 	unsigned long timeout = jiffies + DEFAULT_TIMEOUT_INTERVAL;
272 
273 	while (!(omap_sham_read(dd, offset) & bit)) {
274 		if (time_is_before_jiffies(timeout))
275 			return -ETIMEDOUT;
276 	}
277 
278 	return 0;
279 }
280 
omap_sham_copy_hash_omap2(struct ahash_request * req,int out)281 static void omap_sham_copy_hash_omap2(struct ahash_request *req, int out)
282 {
283 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
284 	struct omap_sham_dev *dd = ctx->dd;
285 	u32 *hash = (u32 *)ctx->digest;
286 	int i;
287 
288 	for (i = 0; i < dd->pdata->digest_size / sizeof(u32); i++) {
289 		if (out)
290 			hash[i] = omap_sham_read(dd, SHA_REG_IDIGEST(dd, i));
291 		else
292 			omap_sham_write(dd, SHA_REG_IDIGEST(dd, i), hash[i]);
293 	}
294 }
295 
omap_sham_copy_hash_omap4(struct ahash_request * req,int out)296 static void omap_sham_copy_hash_omap4(struct ahash_request *req, int out)
297 {
298 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
299 	struct omap_sham_dev *dd = ctx->dd;
300 	int i;
301 
302 	if (ctx->flags & BIT(FLAGS_HMAC)) {
303 		struct crypto_ahash *tfm = crypto_ahash_reqtfm(dd->req);
304 		struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
305 		struct omap_sham_hmac_ctx *bctx = tctx->base;
306 		u32 *opad = (u32 *)bctx->opad;
307 
308 		for (i = 0; i < dd->pdata->digest_size / sizeof(u32); i++) {
309 			if (out)
310 				opad[i] = omap_sham_read(dd,
311 						SHA_REG_ODIGEST(dd, i));
312 			else
313 				omap_sham_write(dd, SHA_REG_ODIGEST(dd, i),
314 						opad[i]);
315 		}
316 	}
317 
318 	omap_sham_copy_hash_omap2(req, out);
319 }
320 
omap_sham_copy_ready_hash(struct ahash_request * req)321 static void omap_sham_copy_ready_hash(struct ahash_request *req)
322 {
323 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
324 	u32 *in = (u32 *)ctx->digest;
325 	u32 *hash = (u32 *)req->result;
326 	int i, d, big_endian = 0;
327 
328 	if (!hash)
329 		return;
330 
331 	switch (ctx->flags & FLAGS_MODE_MASK) {
332 	case FLAGS_MODE_MD5:
333 		d = MD5_DIGEST_SIZE / sizeof(u32);
334 		break;
335 	case FLAGS_MODE_SHA1:
336 		/* OMAP2 SHA1 is big endian */
337 		if (test_bit(FLAGS_BE32_SHA1, &ctx->dd->flags))
338 			big_endian = 1;
339 		d = SHA1_DIGEST_SIZE / sizeof(u32);
340 		break;
341 	case FLAGS_MODE_SHA224:
342 		d = SHA224_DIGEST_SIZE / sizeof(u32);
343 		break;
344 	case FLAGS_MODE_SHA256:
345 		d = SHA256_DIGEST_SIZE / sizeof(u32);
346 		break;
347 	case FLAGS_MODE_SHA384:
348 		d = SHA384_DIGEST_SIZE / sizeof(u32);
349 		break;
350 	case FLAGS_MODE_SHA512:
351 		d = SHA512_DIGEST_SIZE / sizeof(u32);
352 		break;
353 	default:
354 		d = 0;
355 	}
356 
357 	if (big_endian)
358 		for (i = 0; i < d; i++)
359 			put_unaligned(be32_to_cpup((__be32 *)in + i), &hash[i]);
360 	else
361 		for (i = 0; i < d; i++)
362 			put_unaligned(le32_to_cpup((__le32 *)in + i), &hash[i]);
363 }
364 
omap_sham_write_ctrl_omap2(struct omap_sham_dev * dd,size_t length,int final,int dma)365 static void omap_sham_write_ctrl_omap2(struct omap_sham_dev *dd, size_t length,
366 				 int final, int dma)
367 {
368 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
369 	u32 val = length << 5, mask;
370 
371 	if (likely(ctx->digcnt))
372 		omap_sham_write(dd, SHA_REG_DIGCNT(dd), ctx->digcnt);
373 
374 	omap_sham_write_mask(dd, SHA_REG_MASK(dd),
375 		SHA_REG_MASK_IT_EN | (dma ? SHA_REG_MASK_DMA_EN : 0),
376 		SHA_REG_MASK_IT_EN | SHA_REG_MASK_DMA_EN);
377 	/*
378 	 * Setting ALGO_CONST only for the first iteration
379 	 * and CLOSE_HASH only for the last one.
380 	 */
381 	if ((ctx->flags & FLAGS_MODE_MASK) == FLAGS_MODE_SHA1)
382 		val |= SHA_REG_CTRL_ALGO;
383 	if (!ctx->digcnt)
384 		val |= SHA_REG_CTRL_ALGO_CONST;
385 	if (final)
386 		val |= SHA_REG_CTRL_CLOSE_HASH;
387 
388 	mask = SHA_REG_CTRL_ALGO_CONST | SHA_REG_CTRL_CLOSE_HASH |
389 			SHA_REG_CTRL_ALGO | SHA_REG_CTRL_LENGTH;
390 
391 	omap_sham_write_mask(dd, SHA_REG_CTRL, val, mask);
392 }
393 
omap_sham_trigger_omap2(struct omap_sham_dev * dd,size_t length)394 static void omap_sham_trigger_omap2(struct omap_sham_dev *dd, size_t length)
395 {
396 }
397 
omap_sham_poll_irq_omap2(struct omap_sham_dev * dd)398 static int omap_sham_poll_irq_omap2(struct omap_sham_dev *dd)
399 {
400 	return omap_sham_wait(dd, SHA_REG_CTRL, SHA_REG_CTRL_INPUT_READY);
401 }
402 
get_block_size(struct omap_sham_reqctx * ctx)403 static int get_block_size(struct omap_sham_reqctx *ctx)
404 {
405 	int d;
406 
407 	switch (ctx->flags & FLAGS_MODE_MASK) {
408 	case FLAGS_MODE_MD5:
409 	case FLAGS_MODE_SHA1:
410 		d = SHA1_BLOCK_SIZE;
411 		break;
412 	case FLAGS_MODE_SHA224:
413 	case FLAGS_MODE_SHA256:
414 		d = SHA256_BLOCK_SIZE;
415 		break;
416 	case FLAGS_MODE_SHA384:
417 	case FLAGS_MODE_SHA512:
418 		d = SHA512_BLOCK_SIZE;
419 		break;
420 	default:
421 		d = 0;
422 	}
423 
424 	return d;
425 }
426 
omap_sham_write_n(struct omap_sham_dev * dd,u32 offset,u32 * value,int count)427 static void omap_sham_write_n(struct omap_sham_dev *dd, u32 offset,
428 				    u32 *value, int count)
429 {
430 	for (; count--; value++, offset += 4)
431 		omap_sham_write(dd, offset, *value);
432 }
433 
omap_sham_write_ctrl_omap4(struct omap_sham_dev * dd,size_t length,int final,int dma)434 static void omap_sham_write_ctrl_omap4(struct omap_sham_dev *dd, size_t length,
435 				 int final, int dma)
436 {
437 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
438 	u32 val, mask;
439 
440 	if (likely(ctx->digcnt))
441 		omap_sham_write(dd, SHA_REG_DIGCNT(dd), ctx->digcnt);
442 
443 	/*
444 	 * Setting ALGO_CONST only for the first iteration and
445 	 * CLOSE_HASH only for the last one. Note that flags mode bits
446 	 * correspond to algorithm encoding in mode register.
447 	 */
448 	val = (ctx->flags & FLAGS_MODE_MASK) >> (FLAGS_MODE_SHIFT);
449 	if (!ctx->digcnt) {
450 		struct crypto_ahash *tfm = crypto_ahash_reqtfm(dd->req);
451 		struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
452 		struct omap_sham_hmac_ctx *bctx = tctx->base;
453 		int bs, nr_dr;
454 
455 		val |= SHA_REG_MODE_ALGO_CONSTANT;
456 
457 		if (ctx->flags & BIT(FLAGS_HMAC)) {
458 			bs = get_block_size(ctx);
459 			nr_dr = bs / (2 * sizeof(u32));
460 			val |= SHA_REG_MODE_HMAC_KEY_PROC;
461 			omap_sham_write_n(dd, SHA_REG_ODIGEST(dd, 0),
462 					  (u32 *)bctx->ipad, nr_dr);
463 			omap_sham_write_n(dd, SHA_REG_IDIGEST(dd, 0),
464 					  (u32 *)bctx->ipad + nr_dr, nr_dr);
465 			ctx->digcnt += bs;
466 		}
467 	}
468 
469 	if (final) {
470 		val |= SHA_REG_MODE_CLOSE_HASH;
471 
472 		if (ctx->flags & BIT(FLAGS_HMAC))
473 			val |= SHA_REG_MODE_HMAC_OUTER_HASH;
474 	}
475 
476 	mask = SHA_REG_MODE_ALGO_CONSTANT | SHA_REG_MODE_CLOSE_HASH |
477 	       SHA_REG_MODE_ALGO_MASK | SHA_REG_MODE_HMAC_OUTER_HASH |
478 	       SHA_REG_MODE_HMAC_KEY_PROC;
479 
480 	dev_dbg(dd->dev, "ctrl: %08x, flags: %08lx\n", val, ctx->flags);
481 	omap_sham_write_mask(dd, SHA_REG_MODE(dd), val, mask);
482 	omap_sham_write(dd, SHA_REG_IRQENA, SHA_REG_IRQENA_OUTPUT_RDY);
483 	omap_sham_write_mask(dd, SHA_REG_MASK(dd),
484 			     SHA_REG_MASK_IT_EN |
485 				     (dma ? SHA_REG_MASK_DMA_EN : 0),
486 			     SHA_REG_MASK_IT_EN | SHA_REG_MASK_DMA_EN);
487 }
488 
omap_sham_trigger_omap4(struct omap_sham_dev * dd,size_t length)489 static void omap_sham_trigger_omap4(struct omap_sham_dev *dd, size_t length)
490 {
491 	omap_sham_write(dd, SHA_REG_LENGTH(dd), length);
492 }
493 
omap_sham_poll_irq_omap4(struct omap_sham_dev * dd)494 static int omap_sham_poll_irq_omap4(struct omap_sham_dev *dd)
495 {
496 	return omap_sham_wait(dd, SHA_REG_IRQSTATUS,
497 			      SHA_REG_IRQSTATUS_INPUT_RDY);
498 }
499 
omap_sham_xmit_cpu(struct omap_sham_dev * dd,size_t length,int final)500 static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, size_t length,
501 			      int final)
502 {
503 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
504 	int count, len32, bs32, offset = 0;
505 	const u32 *buffer;
506 	int mlen;
507 	struct sg_mapping_iter mi;
508 
509 	dev_dbg(dd->dev, "xmit_cpu: digcnt: %zd, length: %zd, final: %d\n",
510 						ctx->digcnt, length, final);
511 
512 	dd->pdata->write_ctrl(dd, length, final, 0);
513 	dd->pdata->trigger(dd, length);
514 
515 	/* should be non-zero before next lines to disable clocks later */
516 	ctx->digcnt += length;
517 	ctx->total -= length;
518 
519 	if (final)
520 		set_bit(FLAGS_FINAL, &dd->flags); /* catch last interrupt */
521 
522 	set_bit(FLAGS_CPU, &dd->flags);
523 
524 	len32 = DIV_ROUND_UP(length, sizeof(u32));
525 	bs32 = get_block_size(ctx) / sizeof(u32);
526 
527 	sg_miter_start(&mi, ctx->sg, ctx->sg_len,
528 		       SG_MITER_FROM_SG | SG_MITER_ATOMIC);
529 
530 	mlen = 0;
531 
532 	while (len32) {
533 		if (dd->pdata->poll_irq(dd))
534 			return -ETIMEDOUT;
535 
536 		for (count = 0; count < min(len32, bs32); count++, offset++) {
537 			if (!mlen) {
538 				sg_miter_next(&mi);
539 				mlen = mi.length;
540 				if (!mlen) {
541 					pr_err("sg miter failure.\n");
542 					return -EINVAL;
543 				}
544 				offset = 0;
545 				buffer = mi.addr;
546 			}
547 			omap_sham_write(dd, SHA_REG_DIN(dd, count),
548 					buffer[offset]);
549 			mlen -= 4;
550 		}
551 		len32 -= min(len32, bs32);
552 	}
553 
554 	sg_miter_stop(&mi);
555 
556 	return -EINPROGRESS;
557 }
558 
omap_sham_dma_callback(void * param)559 static void omap_sham_dma_callback(void *param)
560 {
561 	struct omap_sham_dev *dd = param;
562 
563 	set_bit(FLAGS_DMA_READY, &dd->flags);
564 	tasklet_schedule(&dd->done_task);
565 }
566 
omap_sham_xmit_dma(struct omap_sham_dev * dd,size_t length,int final)567 static int omap_sham_xmit_dma(struct omap_sham_dev *dd, size_t length,
568 			      int final)
569 {
570 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
571 	struct dma_async_tx_descriptor *tx;
572 	struct dma_slave_config cfg;
573 	int ret;
574 
575 	dev_dbg(dd->dev, "xmit_dma: digcnt: %zd, length: %zd, final: %d\n",
576 						ctx->digcnt, length, final);
577 
578 	if (!dma_map_sg(dd->dev, ctx->sg, ctx->sg_len, DMA_TO_DEVICE)) {
579 		dev_err(dd->dev, "dma_map_sg error\n");
580 		return -EINVAL;
581 	}
582 
583 	memset(&cfg, 0, sizeof(cfg));
584 
585 	cfg.dst_addr = dd->phys_base + SHA_REG_DIN(dd, 0);
586 	cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
587 	cfg.dst_maxburst = get_block_size(ctx) / DMA_SLAVE_BUSWIDTH_4_BYTES;
588 
589 	ret = dmaengine_slave_config(dd->dma_lch, &cfg);
590 	if (ret) {
591 		pr_err("omap-sham: can't configure dmaengine slave: %d\n", ret);
592 		return ret;
593 	}
594 
595 	tx = dmaengine_prep_slave_sg(dd->dma_lch, ctx->sg, ctx->sg_len,
596 				     DMA_MEM_TO_DEV,
597 				     DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
598 
599 	if (!tx) {
600 		dev_err(dd->dev, "prep_slave_sg failed\n");
601 		return -EINVAL;
602 	}
603 
604 	tx->callback = omap_sham_dma_callback;
605 	tx->callback_param = dd;
606 
607 	dd->pdata->write_ctrl(dd, length, final, 1);
608 
609 	ctx->digcnt += length;
610 	ctx->total -= length;
611 
612 	if (final)
613 		set_bit(FLAGS_FINAL, &dd->flags); /* catch last interrupt */
614 
615 	set_bit(FLAGS_DMA_ACTIVE, &dd->flags);
616 
617 	dmaengine_submit(tx);
618 	dma_async_issue_pending(dd->dma_lch);
619 
620 	dd->pdata->trigger(dd, length);
621 
622 	return -EINPROGRESS;
623 }
624 
omap_sham_copy_sg_lists(struct omap_sham_reqctx * ctx,struct scatterlist * sg,int bs,int new_len)625 static int omap_sham_copy_sg_lists(struct omap_sham_reqctx *ctx,
626 				   struct scatterlist *sg, int bs, int new_len)
627 {
628 	int n = sg_nents(sg);
629 	struct scatterlist *tmp;
630 	int offset = ctx->offset;
631 
632 	ctx->total = new_len;
633 
634 	if (ctx->bufcnt)
635 		n++;
636 
637 	ctx->sg = kmalloc_array(n, sizeof(*sg), GFP_KERNEL);
638 	if (!ctx->sg)
639 		return -ENOMEM;
640 
641 	sg_init_table(ctx->sg, n);
642 
643 	tmp = ctx->sg;
644 
645 	ctx->sg_len = 0;
646 
647 	if (ctx->bufcnt) {
648 		sg_set_buf(tmp, ctx->dd->xmit_buf, ctx->bufcnt);
649 		tmp = sg_next(tmp);
650 		ctx->sg_len++;
651 		new_len -= ctx->bufcnt;
652 	}
653 
654 	while (sg && new_len) {
655 		int len = sg->length - offset;
656 
657 		if (len <= 0) {
658 			offset -= sg->length;
659 			sg = sg_next(sg);
660 			continue;
661 		}
662 
663 		if (new_len < len)
664 			len = new_len;
665 
666 		if (len > 0) {
667 			new_len -= len;
668 			sg_set_page(tmp, sg_page(sg), len, sg->offset + offset);
669 			offset = 0;
670 			ctx->offset = 0;
671 			ctx->sg_len++;
672 			if (new_len <= 0)
673 				break;
674 			tmp = sg_next(tmp);
675 		}
676 
677 		sg = sg_next(sg);
678 	}
679 
680 	if (tmp)
681 		sg_mark_end(tmp);
682 
683 	set_bit(FLAGS_SGS_ALLOCED, &ctx->dd->flags);
684 
685 	ctx->offset += new_len - ctx->bufcnt;
686 	ctx->bufcnt = 0;
687 
688 	return 0;
689 }
690 
omap_sham_copy_sgs(struct omap_sham_reqctx * ctx,struct scatterlist * sg,int bs,unsigned int new_len)691 static int omap_sham_copy_sgs(struct omap_sham_reqctx *ctx,
692 			      struct scatterlist *sg, int bs,
693 			      unsigned int new_len)
694 {
695 	int pages;
696 	void *buf;
697 
698 	pages = get_order(new_len);
699 
700 	buf = (void *)__get_free_pages(GFP_ATOMIC, pages);
701 	if (!buf) {
702 		pr_err("Couldn't allocate pages for unaligned cases.\n");
703 		return -ENOMEM;
704 	}
705 
706 	if (ctx->bufcnt)
707 		memcpy(buf, ctx->dd->xmit_buf, ctx->bufcnt);
708 
709 	scatterwalk_map_and_copy(buf + ctx->bufcnt, sg, ctx->offset,
710 				 min(new_len, ctx->total) - ctx->bufcnt, 0);
711 	sg_init_table(ctx->sgl, 1);
712 	sg_set_buf(ctx->sgl, buf, new_len);
713 	ctx->sg = ctx->sgl;
714 	set_bit(FLAGS_SGS_COPIED, &ctx->dd->flags);
715 	ctx->sg_len = 1;
716 	ctx->offset += new_len - ctx->bufcnt;
717 	ctx->bufcnt = 0;
718 	ctx->total = new_len;
719 
720 	return 0;
721 }
722 
omap_sham_align_sgs(struct scatterlist * sg,int nbytes,int bs,bool final,struct omap_sham_reqctx * rctx)723 static int omap_sham_align_sgs(struct scatterlist *sg,
724 			       int nbytes, int bs, bool final,
725 			       struct omap_sham_reqctx *rctx)
726 {
727 	int n = 0;
728 	bool aligned = true;
729 	bool list_ok = true;
730 	struct scatterlist *sg_tmp = sg;
731 	int new_len;
732 	int offset = rctx->offset;
733 	int bufcnt = rctx->bufcnt;
734 
735 	if (!sg || !sg->length || !nbytes) {
736 		if (bufcnt) {
737 			bufcnt = DIV_ROUND_UP(bufcnt, bs) * bs;
738 			sg_init_table(rctx->sgl, 1);
739 			sg_set_buf(rctx->sgl, rctx->dd->xmit_buf, bufcnt);
740 			rctx->sg = rctx->sgl;
741 			rctx->sg_len = 1;
742 		}
743 
744 		return 0;
745 	}
746 
747 	new_len = nbytes;
748 
749 	if (offset)
750 		list_ok = false;
751 
752 	if (final)
753 		new_len = DIV_ROUND_UP(new_len, bs) * bs;
754 	else
755 		new_len = (new_len - 1) / bs * bs;
756 
757 	if (!new_len)
758 		return 0;
759 
760 	if (nbytes != new_len)
761 		list_ok = false;
762 
763 	while (nbytes > 0 && sg_tmp) {
764 		n++;
765 
766 		if (bufcnt) {
767 			if (!IS_ALIGNED(bufcnt, bs)) {
768 				aligned = false;
769 				break;
770 			}
771 			nbytes -= bufcnt;
772 			bufcnt = 0;
773 			if (!nbytes)
774 				list_ok = false;
775 
776 			continue;
777 		}
778 
779 #ifdef CONFIG_ZONE_DMA
780 		if (page_zonenum(sg_page(sg_tmp)) != ZONE_DMA) {
781 			aligned = false;
782 			break;
783 		}
784 #endif
785 
786 		if (offset < sg_tmp->length) {
787 			if (!IS_ALIGNED(offset + sg_tmp->offset, 4)) {
788 				aligned = false;
789 				break;
790 			}
791 
792 			if (!IS_ALIGNED(sg_tmp->length - offset, bs)) {
793 				aligned = false;
794 				break;
795 			}
796 		}
797 
798 		if (offset) {
799 			offset -= sg_tmp->length;
800 			if (offset < 0) {
801 				nbytes += offset;
802 				offset = 0;
803 			}
804 		} else {
805 			nbytes -= sg_tmp->length;
806 		}
807 
808 		sg_tmp = sg_next(sg_tmp);
809 
810 		if (nbytes < 0) {
811 			list_ok = false;
812 			break;
813 		}
814 	}
815 
816 	if (new_len > OMAP_SHA_MAX_DMA_LEN) {
817 		new_len = OMAP_SHA_MAX_DMA_LEN;
818 		aligned = false;
819 	}
820 
821 	if (!aligned)
822 		return omap_sham_copy_sgs(rctx, sg, bs, new_len);
823 	else if (!list_ok)
824 		return omap_sham_copy_sg_lists(rctx, sg, bs, new_len);
825 
826 	rctx->total = new_len;
827 	rctx->offset += new_len;
828 	rctx->sg_len = n;
829 	if (rctx->bufcnt) {
830 		sg_init_table(rctx->sgl, 2);
831 		sg_set_buf(rctx->sgl, rctx->dd->xmit_buf, rctx->bufcnt);
832 		sg_chain(rctx->sgl, 2, sg);
833 		rctx->sg = rctx->sgl;
834 	} else {
835 		rctx->sg = sg;
836 	}
837 
838 	return 0;
839 }
840 
omap_sham_prepare_request(struct crypto_engine * engine,void * areq)841 static int omap_sham_prepare_request(struct crypto_engine *engine, void *areq)
842 {
843 	struct ahash_request *req = container_of(areq, struct ahash_request,
844 						 base);
845 	struct omap_sham_reqctx *rctx = ahash_request_ctx(req);
846 	int bs;
847 	int ret;
848 	unsigned int nbytes;
849 	bool final = rctx->flags & BIT(FLAGS_FINUP);
850 	bool update = rctx->op == OP_UPDATE;
851 	int hash_later;
852 
853 	bs = get_block_size(rctx);
854 
855 	nbytes = rctx->bufcnt;
856 
857 	if (update)
858 		nbytes += req->nbytes - rctx->offset;
859 
860 	dev_dbg(rctx->dd->dev,
861 		"%s: nbytes=%d, bs=%d, total=%d, offset=%d, bufcnt=%zd\n",
862 		__func__, nbytes, bs, rctx->total, rctx->offset,
863 		rctx->bufcnt);
864 
865 	if (!nbytes)
866 		return 0;
867 
868 	rctx->total = nbytes;
869 
870 	if (update && req->nbytes && (!IS_ALIGNED(rctx->bufcnt, bs))) {
871 		int len = bs - rctx->bufcnt % bs;
872 
873 		if (len > req->nbytes)
874 			len = req->nbytes;
875 		scatterwalk_map_and_copy(rctx->buffer + rctx->bufcnt, req->src,
876 					 0, len, 0);
877 		rctx->bufcnt += len;
878 		rctx->offset = len;
879 	}
880 
881 	if (rctx->bufcnt)
882 		memcpy(rctx->dd->xmit_buf, rctx->buffer, rctx->bufcnt);
883 
884 	ret = omap_sham_align_sgs(req->src, nbytes, bs, final, rctx);
885 	if (ret)
886 		return ret;
887 
888 	hash_later = nbytes - rctx->total;
889 	if (hash_later < 0)
890 		hash_later = 0;
891 
892 	if (hash_later && hash_later <= rctx->buflen) {
893 		scatterwalk_map_and_copy(rctx->buffer,
894 					 req->src,
895 					 req->nbytes - hash_later,
896 					 hash_later, 0);
897 
898 		rctx->bufcnt = hash_later;
899 	} else {
900 		rctx->bufcnt = 0;
901 	}
902 
903 	if (hash_later > rctx->buflen)
904 		set_bit(FLAGS_HUGE, &rctx->dd->flags);
905 
906 	rctx->total = min(nbytes, rctx->total);
907 
908 	return 0;
909 }
910 
omap_sham_update_dma_stop(struct omap_sham_dev * dd)911 static int omap_sham_update_dma_stop(struct omap_sham_dev *dd)
912 {
913 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
914 
915 	dma_unmap_sg(dd->dev, ctx->sg, ctx->sg_len, DMA_TO_DEVICE);
916 
917 	clear_bit(FLAGS_DMA_ACTIVE, &dd->flags);
918 
919 	return 0;
920 }
921 
omap_sham_find_dev(struct omap_sham_reqctx * ctx)922 static struct omap_sham_dev *omap_sham_find_dev(struct omap_sham_reqctx *ctx)
923 {
924 	struct omap_sham_dev *dd;
925 
926 	if (ctx->dd)
927 		return ctx->dd;
928 
929 	spin_lock_bh(&sham.lock);
930 	dd = list_first_entry(&sham.dev_list, struct omap_sham_dev, list);
931 	list_move_tail(&dd->list, &sham.dev_list);
932 	ctx->dd = dd;
933 	spin_unlock_bh(&sham.lock);
934 
935 	return dd;
936 }
937 
omap_sham_init(struct ahash_request * req)938 static int omap_sham_init(struct ahash_request *req)
939 {
940 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
941 	struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
942 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
943 	struct omap_sham_dev *dd;
944 	int bs = 0;
945 
946 	ctx->dd = NULL;
947 
948 	dd = omap_sham_find_dev(ctx);
949 	if (!dd)
950 		return -ENODEV;
951 
952 	ctx->flags = 0;
953 
954 	dev_dbg(dd->dev, "init: digest size: %d\n",
955 		crypto_ahash_digestsize(tfm));
956 
957 	switch (crypto_ahash_digestsize(tfm)) {
958 	case MD5_DIGEST_SIZE:
959 		ctx->flags |= FLAGS_MODE_MD5;
960 		bs = SHA1_BLOCK_SIZE;
961 		break;
962 	case SHA1_DIGEST_SIZE:
963 		ctx->flags |= FLAGS_MODE_SHA1;
964 		bs = SHA1_BLOCK_SIZE;
965 		break;
966 	case SHA224_DIGEST_SIZE:
967 		ctx->flags |= FLAGS_MODE_SHA224;
968 		bs = SHA224_BLOCK_SIZE;
969 		break;
970 	case SHA256_DIGEST_SIZE:
971 		ctx->flags |= FLAGS_MODE_SHA256;
972 		bs = SHA256_BLOCK_SIZE;
973 		break;
974 	case SHA384_DIGEST_SIZE:
975 		ctx->flags |= FLAGS_MODE_SHA384;
976 		bs = SHA384_BLOCK_SIZE;
977 		break;
978 	case SHA512_DIGEST_SIZE:
979 		ctx->flags |= FLAGS_MODE_SHA512;
980 		bs = SHA512_BLOCK_SIZE;
981 		break;
982 	}
983 
984 	ctx->bufcnt = 0;
985 	ctx->digcnt = 0;
986 	ctx->total = 0;
987 	ctx->offset = 0;
988 	ctx->buflen = BUFLEN;
989 
990 	if (tctx->flags & BIT(FLAGS_HMAC)) {
991 		if (!test_bit(FLAGS_AUTO_XOR, &dd->flags)) {
992 			struct omap_sham_hmac_ctx *bctx = tctx->base;
993 
994 			memcpy(ctx->buffer, bctx->ipad, bs);
995 			ctx->bufcnt = bs;
996 		}
997 
998 		ctx->flags |= BIT(FLAGS_HMAC);
999 	}
1000 
1001 	return 0;
1002 
1003 }
1004 
omap_sham_update_req(struct omap_sham_dev * dd)1005 static int omap_sham_update_req(struct omap_sham_dev *dd)
1006 {
1007 	struct ahash_request *req = dd->req;
1008 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1009 	int err;
1010 	bool final = (ctx->flags & BIT(FLAGS_FINUP)) &&
1011 		!(dd->flags & BIT(FLAGS_HUGE));
1012 
1013 	dev_dbg(dd->dev, "update_req: total: %u, digcnt: %zd, final: %d",
1014 		ctx->total, ctx->digcnt, final);
1015 
1016 	if (ctx->total < get_block_size(ctx) ||
1017 	    ctx->total < dd->fallback_sz)
1018 		ctx->flags |= BIT(FLAGS_CPU);
1019 
1020 	if (ctx->flags & BIT(FLAGS_CPU))
1021 		err = omap_sham_xmit_cpu(dd, ctx->total, final);
1022 	else
1023 		err = omap_sham_xmit_dma(dd, ctx->total, final);
1024 
1025 	/* wait for dma completion before can take more data */
1026 	dev_dbg(dd->dev, "update: err: %d, digcnt: %zd\n", err, ctx->digcnt);
1027 
1028 	return err;
1029 }
1030 
omap_sham_final_req(struct omap_sham_dev * dd)1031 static int omap_sham_final_req(struct omap_sham_dev *dd)
1032 {
1033 	struct ahash_request *req = dd->req;
1034 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1035 	int err = 0, use_dma = 1;
1036 
1037 	if (dd->flags & BIT(FLAGS_HUGE))
1038 		return 0;
1039 
1040 	if ((ctx->total <= get_block_size(ctx)) || dd->polling_mode)
1041 		/*
1042 		 * faster to handle last block with cpu or
1043 		 * use cpu when dma is not present.
1044 		 */
1045 		use_dma = 0;
1046 
1047 	if (use_dma)
1048 		err = omap_sham_xmit_dma(dd, ctx->total, 1);
1049 	else
1050 		err = omap_sham_xmit_cpu(dd, ctx->total, 1);
1051 
1052 	ctx->bufcnt = 0;
1053 
1054 	dev_dbg(dd->dev, "final_req: err: %d\n", err);
1055 
1056 	return err;
1057 }
1058 
omap_sham_hash_one_req(struct crypto_engine * engine,void * areq)1059 static int omap_sham_hash_one_req(struct crypto_engine *engine, void *areq)
1060 {
1061 	struct ahash_request *req = container_of(areq, struct ahash_request,
1062 						 base);
1063 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1064 	struct omap_sham_dev *dd = ctx->dd;
1065 	int err;
1066 	bool final = (ctx->flags & BIT(FLAGS_FINUP)) &&
1067 			!(dd->flags & BIT(FLAGS_HUGE));
1068 
1069 	dev_dbg(dd->dev, "hash-one: op: %u, total: %u, digcnt: %zd, final: %d",
1070 		ctx->op, ctx->total, ctx->digcnt, final);
1071 
1072 	err = omap_sham_prepare_request(engine, areq);
1073 	if (err)
1074 		return err;
1075 
1076 	err = pm_runtime_resume_and_get(dd->dev);
1077 	if (err < 0) {
1078 		dev_err(dd->dev, "failed to get sync: %d\n", err);
1079 		return err;
1080 	}
1081 
1082 	dd->err = 0;
1083 	dd->req = req;
1084 
1085 	if (ctx->digcnt)
1086 		dd->pdata->copy_hash(req, 0);
1087 
1088 	if (ctx->op == OP_UPDATE)
1089 		err = omap_sham_update_req(dd);
1090 	else if (ctx->op == OP_FINAL)
1091 		err = omap_sham_final_req(dd);
1092 
1093 	if (err != -EINPROGRESS)
1094 		omap_sham_finish_req(req, err);
1095 
1096 	return 0;
1097 }
1098 
omap_sham_finish_hmac(struct ahash_request * req)1099 static int omap_sham_finish_hmac(struct ahash_request *req)
1100 {
1101 	struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
1102 	struct omap_sham_hmac_ctx *bctx = tctx->base;
1103 	int bs = crypto_shash_blocksize(bctx->shash);
1104 	int ds = crypto_shash_digestsize(bctx->shash);
1105 	SHASH_DESC_ON_STACK(shash, bctx->shash);
1106 
1107 	shash->tfm = bctx->shash;
1108 
1109 	return crypto_shash_init(shash) ?:
1110 	       crypto_shash_update(shash, bctx->opad, bs) ?:
1111 	       crypto_shash_finup(shash, req->result, ds, req->result);
1112 }
1113 
omap_sham_finish(struct ahash_request * req)1114 static int omap_sham_finish(struct ahash_request *req)
1115 {
1116 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1117 	struct omap_sham_dev *dd = ctx->dd;
1118 	int err = 0;
1119 
1120 	if (ctx->digcnt) {
1121 		omap_sham_copy_ready_hash(req);
1122 		if ((ctx->flags & BIT(FLAGS_HMAC)) &&
1123 				!test_bit(FLAGS_AUTO_XOR, &dd->flags))
1124 			err = omap_sham_finish_hmac(req);
1125 	}
1126 
1127 	dev_dbg(dd->dev, "digcnt: %zd, bufcnt: %zd\n", ctx->digcnt, ctx->bufcnt);
1128 
1129 	return err;
1130 }
1131 
omap_sham_finish_req(struct ahash_request * req,int err)1132 static void omap_sham_finish_req(struct ahash_request *req, int err)
1133 {
1134 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1135 	struct omap_sham_dev *dd = ctx->dd;
1136 
1137 	if (test_bit(FLAGS_SGS_COPIED, &dd->flags))
1138 		free_pages((unsigned long)sg_virt(ctx->sg),
1139 			   get_order(ctx->sg->length));
1140 
1141 	if (test_bit(FLAGS_SGS_ALLOCED, &dd->flags))
1142 		kfree(ctx->sg);
1143 
1144 	ctx->sg = NULL;
1145 
1146 	dd->flags &= ~(BIT(FLAGS_SGS_ALLOCED) | BIT(FLAGS_SGS_COPIED) |
1147 		       BIT(FLAGS_CPU) | BIT(FLAGS_DMA_READY) |
1148 		       BIT(FLAGS_OUTPUT_READY));
1149 
1150 	if (!err)
1151 		dd->pdata->copy_hash(req, 1);
1152 
1153 	if (dd->flags & BIT(FLAGS_HUGE)) {
1154 		/* Re-enqueue the request */
1155 		omap_sham_enqueue(req, ctx->op);
1156 		return;
1157 	}
1158 
1159 	if (!err) {
1160 		if (test_bit(FLAGS_FINAL, &dd->flags))
1161 			err = omap_sham_finish(req);
1162 	} else {
1163 		ctx->flags |= BIT(FLAGS_ERROR);
1164 	}
1165 
1166 	/* atomic operation is not needed here */
1167 	dd->flags &= ~(BIT(FLAGS_FINAL) | BIT(FLAGS_CPU) |
1168 			BIT(FLAGS_DMA_READY) | BIT(FLAGS_OUTPUT_READY));
1169 
1170 	pm_runtime_mark_last_busy(dd->dev);
1171 	pm_runtime_put_autosuspend(dd->dev);
1172 
1173 	ctx->offset = 0;
1174 
1175 	crypto_finalize_hash_request(dd->engine, req, err);
1176 }
1177 
omap_sham_handle_queue(struct omap_sham_dev * dd,struct ahash_request * req)1178 static int omap_sham_handle_queue(struct omap_sham_dev *dd,
1179 				  struct ahash_request *req)
1180 {
1181 	return crypto_transfer_hash_request_to_engine(dd->engine, req);
1182 }
1183 
omap_sham_enqueue(struct ahash_request * req,unsigned int op)1184 static int omap_sham_enqueue(struct ahash_request *req, unsigned int op)
1185 {
1186 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1187 	struct omap_sham_dev *dd = ctx->dd;
1188 
1189 	ctx->op = op;
1190 
1191 	return omap_sham_handle_queue(dd, req);
1192 }
1193 
omap_sham_update(struct ahash_request * req)1194 static int omap_sham_update(struct ahash_request *req)
1195 {
1196 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1197 	struct omap_sham_dev *dd = omap_sham_find_dev(ctx);
1198 
1199 	if (!req->nbytes)
1200 		return 0;
1201 
1202 	if (ctx->bufcnt + req->nbytes <= ctx->buflen) {
1203 		scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, req->src,
1204 					 0, req->nbytes, 0);
1205 		ctx->bufcnt += req->nbytes;
1206 		return 0;
1207 	}
1208 
1209 	if (dd->polling_mode)
1210 		ctx->flags |= BIT(FLAGS_CPU);
1211 
1212 	return omap_sham_enqueue(req, OP_UPDATE);
1213 }
1214 
omap_sham_final_shash(struct ahash_request * req)1215 static int omap_sham_final_shash(struct ahash_request *req)
1216 {
1217 	struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
1218 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1219 	int offset = 0;
1220 
1221 	/*
1222 	 * If we are running HMAC on limited hardware support, skip
1223 	 * the ipad in the beginning of the buffer if we are going for
1224 	 * software fallback algorithm.
1225 	 */
1226 	if (test_bit(FLAGS_HMAC, &ctx->flags) &&
1227 	    !test_bit(FLAGS_AUTO_XOR, &ctx->dd->flags))
1228 		offset = get_block_size(ctx);
1229 
1230 	return crypto_shash_tfm_digest(tctx->fallback, ctx->buffer + offset,
1231 				       ctx->bufcnt - offset, req->result);
1232 }
1233 
omap_sham_final(struct ahash_request * req)1234 static int omap_sham_final(struct ahash_request *req)
1235 {
1236 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1237 
1238 	ctx->flags |= BIT(FLAGS_FINUP);
1239 
1240 	if (ctx->flags & BIT(FLAGS_ERROR))
1241 		return 0; /* uncompleted hash is not needed */
1242 
1243 	/*
1244 	 * OMAP HW accel works only with buffers >= 9.
1245 	 * HMAC is always >= 9 because ipad == block size.
1246 	 * If buffersize is less than fallback_sz, we use fallback
1247 	 * SW encoding, as using DMA + HW in this case doesn't provide
1248 	 * any benefit.
1249 	 */
1250 	if (!ctx->digcnt && ctx->bufcnt < ctx->dd->fallback_sz)
1251 		return omap_sham_final_shash(req);
1252 	else if (ctx->bufcnt)
1253 		return omap_sham_enqueue(req, OP_FINAL);
1254 
1255 	/* copy ready hash (+ finalize hmac) */
1256 	return omap_sham_finish(req);
1257 }
1258 
omap_sham_finup(struct ahash_request * req)1259 static int omap_sham_finup(struct ahash_request *req)
1260 {
1261 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1262 	int err1, err2;
1263 
1264 	ctx->flags |= BIT(FLAGS_FINUP);
1265 
1266 	err1 = omap_sham_update(req);
1267 	if (err1 == -EINPROGRESS || err1 == -EBUSY)
1268 		return err1;
1269 	/*
1270 	 * final() has to be always called to cleanup resources
1271 	 * even if udpate() failed, except EINPROGRESS
1272 	 */
1273 	err2 = omap_sham_final(req);
1274 
1275 	return err1 ?: err2;
1276 }
1277 
omap_sham_digest(struct ahash_request * req)1278 static int omap_sham_digest(struct ahash_request *req)
1279 {
1280 	return omap_sham_init(req) ?: omap_sham_finup(req);
1281 }
1282 
omap_sham_setkey(struct crypto_ahash * tfm,const u8 * key,unsigned int keylen)1283 static int omap_sham_setkey(struct crypto_ahash *tfm, const u8 *key,
1284 		      unsigned int keylen)
1285 {
1286 	struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
1287 	struct omap_sham_hmac_ctx *bctx = tctx->base;
1288 	int bs = crypto_shash_blocksize(bctx->shash);
1289 	int ds = crypto_shash_digestsize(bctx->shash);
1290 	int err, i;
1291 
1292 	err = crypto_shash_setkey(tctx->fallback, key, keylen);
1293 	if (err)
1294 		return err;
1295 
1296 	if (keylen > bs) {
1297 		err = crypto_shash_tfm_digest(bctx->shash, key, keylen,
1298 					      bctx->ipad);
1299 		if (err)
1300 			return err;
1301 		keylen = ds;
1302 	} else {
1303 		memcpy(bctx->ipad, key, keylen);
1304 	}
1305 
1306 	memset(bctx->ipad + keylen, 0, bs - keylen);
1307 
1308 	if (!test_bit(FLAGS_AUTO_XOR, &sham.flags)) {
1309 		memcpy(bctx->opad, bctx->ipad, bs);
1310 
1311 		for (i = 0; i < bs; i++) {
1312 			bctx->ipad[i] ^= HMAC_IPAD_VALUE;
1313 			bctx->opad[i] ^= HMAC_OPAD_VALUE;
1314 		}
1315 	}
1316 
1317 	return err;
1318 }
1319 
omap_sham_cra_init_alg(struct crypto_tfm * tfm,const char * alg_base)1320 static int omap_sham_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base)
1321 {
1322 	struct omap_sham_ctx *tctx = crypto_tfm_ctx(tfm);
1323 	const char *alg_name = crypto_tfm_alg_name(tfm);
1324 
1325 	/* Allocate a fallback and abort if it failed. */
1326 	tctx->fallback = crypto_alloc_shash(alg_name, 0,
1327 					    CRYPTO_ALG_NEED_FALLBACK);
1328 	if (IS_ERR(tctx->fallback)) {
1329 		pr_err("omap-sham: fallback driver '%s' "
1330 				"could not be loaded.\n", alg_name);
1331 		return PTR_ERR(tctx->fallback);
1332 	}
1333 
1334 	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1335 				 sizeof(struct omap_sham_reqctx) + BUFLEN);
1336 
1337 	if (alg_base) {
1338 		struct omap_sham_hmac_ctx *bctx = tctx->base;
1339 		tctx->flags |= BIT(FLAGS_HMAC);
1340 		bctx->shash = crypto_alloc_shash(alg_base, 0,
1341 						CRYPTO_ALG_NEED_FALLBACK);
1342 		if (IS_ERR(bctx->shash)) {
1343 			pr_err("omap-sham: base driver '%s' "
1344 					"could not be loaded.\n", alg_base);
1345 			crypto_free_shash(tctx->fallback);
1346 			return PTR_ERR(bctx->shash);
1347 		}
1348 
1349 	}
1350 
1351 	return 0;
1352 }
1353 
omap_sham_cra_init(struct crypto_tfm * tfm)1354 static int omap_sham_cra_init(struct crypto_tfm *tfm)
1355 {
1356 	return omap_sham_cra_init_alg(tfm, NULL);
1357 }
1358 
omap_sham_cra_sha1_init(struct crypto_tfm * tfm)1359 static int omap_sham_cra_sha1_init(struct crypto_tfm *tfm)
1360 {
1361 	return omap_sham_cra_init_alg(tfm, "sha1");
1362 }
1363 
omap_sham_cra_sha224_init(struct crypto_tfm * tfm)1364 static int omap_sham_cra_sha224_init(struct crypto_tfm *tfm)
1365 {
1366 	return omap_sham_cra_init_alg(tfm, "sha224");
1367 }
1368 
omap_sham_cra_sha256_init(struct crypto_tfm * tfm)1369 static int omap_sham_cra_sha256_init(struct crypto_tfm *tfm)
1370 {
1371 	return omap_sham_cra_init_alg(tfm, "sha256");
1372 }
1373 
omap_sham_cra_md5_init(struct crypto_tfm * tfm)1374 static int omap_sham_cra_md5_init(struct crypto_tfm *tfm)
1375 {
1376 	return omap_sham_cra_init_alg(tfm, "md5");
1377 }
1378 
omap_sham_cra_sha384_init(struct crypto_tfm * tfm)1379 static int omap_sham_cra_sha384_init(struct crypto_tfm *tfm)
1380 {
1381 	return omap_sham_cra_init_alg(tfm, "sha384");
1382 }
1383 
omap_sham_cra_sha512_init(struct crypto_tfm * tfm)1384 static int omap_sham_cra_sha512_init(struct crypto_tfm *tfm)
1385 {
1386 	return omap_sham_cra_init_alg(tfm, "sha512");
1387 }
1388 
omap_sham_cra_exit(struct crypto_tfm * tfm)1389 static void omap_sham_cra_exit(struct crypto_tfm *tfm)
1390 {
1391 	struct omap_sham_ctx *tctx = crypto_tfm_ctx(tfm);
1392 
1393 	crypto_free_shash(tctx->fallback);
1394 	tctx->fallback = NULL;
1395 
1396 	if (tctx->flags & BIT(FLAGS_HMAC)) {
1397 		struct omap_sham_hmac_ctx *bctx = tctx->base;
1398 		crypto_free_shash(bctx->shash);
1399 	}
1400 }
1401 
omap_sham_export(struct ahash_request * req,void * out)1402 static int omap_sham_export(struct ahash_request *req, void *out)
1403 {
1404 	struct omap_sham_reqctx *rctx = ahash_request_ctx(req);
1405 
1406 	memcpy(out, rctx, sizeof(*rctx) + rctx->bufcnt);
1407 
1408 	return 0;
1409 }
1410 
omap_sham_import(struct ahash_request * req,const void * in)1411 static int omap_sham_import(struct ahash_request *req, const void *in)
1412 {
1413 	struct omap_sham_reqctx *rctx = ahash_request_ctx(req);
1414 	const struct omap_sham_reqctx *ctx_in = in;
1415 
1416 	memcpy(rctx, in, sizeof(*rctx) + ctx_in->bufcnt);
1417 
1418 	return 0;
1419 }
1420 
1421 static struct ahash_engine_alg algs_sha1_md5[] = {
1422 {
1423 	.base.init		= omap_sham_init,
1424 	.base.update		= omap_sham_update,
1425 	.base.final		= omap_sham_final,
1426 	.base.finup		= omap_sham_finup,
1427 	.base.digest		= omap_sham_digest,
1428 	.base.halg.digestsize	= SHA1_DIGEST_SIZE,
1429 	.base.halg.base	= {
1430 		.cra_name		= "sha1",
1431 		.cra_driver_name	= "omap-sha1",
1432 		.cra_priority		= 400,
1433 		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1434 						CRYPTO_ALG_ASYNC |
1435 						CRYPTO_ALG_NEED_FALLBACK,
1436 		.cra_blocksize		= SHA1_BLOCK_SIZE,
1437 		.cra_ctxsize		= sizeof(struct omap_sham_ctx),
1438 		.cra_module		= THIS_MODULE,
1439 		.cra_init		= omap_sham_cra_init,
1440 		.cra_exit		= omap_sham_cra_exit,
1441 	},
1442 	.op.do_one_request = omap_sham_hash_one_req,
1443 },
1444 {
1445 	.base.init		= omap_sham_init,
1446 	.base.update		= omap_sham_update,
1447 	.base.final		= omap_sham_final,
1448 	.base.finup		= omap_sham_finup,
1449 	.base.digest		= omap_sham_digest,
1450 	.base.halg.digestsize	= MD5_DIGEST_SIZE,
1451 	.base.halg.base	= {
1452 		.cra_name		= "md5",
1453 		.cra_driver_name	= "omap-md5",
1454 		.cra_priority		= 400,
1455 		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1456 						CRYPTO_ALG_ASYNC |
1457 						CRYPTO_ALG_NEED_FALLBACK,
1458 		.cra_blocksize		= SHA1_BLOCK_SIZE,
1459 		.cra_ctxsize		= sizeof(struct omap_sham_ctx),
1460 		.cra_module		= THIS_MODULE,
1461 		.cra_init		= omap_sham_cra_init,
1462 		.cra_exit		= omap_sham_cra_exit,
1463 	},
1464 	.op.do_one_request = omap_sham_hash_one_req,
1465 },
1466 {
1467 	.base.init		= omap_sham_init,
1468 	.base.update		= omap_sham_update,
1469 	.base.final		= omap_sham_final,
1470 	.base.finup		= omap_sham_finup,
1471 	.base.digest		= omap_sham_digest,
1472 	.base.setkey		= omap_sham_setkey,
1473 	.base.halg.digestsize	= SHA1_DIGEST_SIZE,
1474 	.base.halg.base	= {
1475 		.cra_name		= "hmac(sha1)",
1476 		.cra_driver_name	= "omap-hmac-sha1",
1477 		.cra_priority		= 400,
1478 		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1479 						CRYPTO_ALG_ASYNC |
1480 						CRYPTO_ALG_NEED_FALLBACK,
1481 		.cra_blocksize		= SHA1_BLOCK_SIZE,
1482 		.cra_ctxsize		= sizeof(struct omap_sham_ctx) +
1483 					sizeof(struct omap_sham_hmac_ctx),
1484 		.cra_module		= THIS_MODULE,
1485 		.cra_init		= omap_sham_cra_sha1_init,
1486 		.cra_exit		= omap_sham_cra_exit,
1487 	},
1488 	.op.do_one_request = omap_sham_hash_one_req,
1489 },
1490 {
1491 	.base.init		= omap_sham_init,
1492 	.base.update		= omap_sham_update,
1493 	.base.final		= omap_sham_final,
1494 	.base.finup		= omap_sham_finup,
1495 	.base.digest		= omap_sham_digest,
1496 	.base.setkey		= omap_sham_setkey,
1497 	.base.halg.digestsize	= MD5_DIGEST_SIZE,
1498 	.base.halg.base	= {
1499 		.cra_name		= "hmac(md5)",
1500 		.cra_driver_name	= "omap-hmac-md5",
1501 		.cra_priority		= 400,
1502 		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1503 						CRYPTO_ALG_ASYNC |
1504 						CRYPTO_ALG_NEED_FALLBACK,
1505 		.cra_blocksize		= SHA1_BLOCK_SIZE,
1506 		.cra_ctxsize		= sizeof(struct omap_sham_ctx) +
1507 					sizeof(struct omap_sham_hmac_ctx),
1508 		.cra_module		= THIS_MODULE,
1509 		.cra_init		= omap_sham_cra_md5_init,
1510 		.cra_exit		= omap_sham_cra_exit,
1511 	},
1512 	.op.do_one_request = omap_sham_hash_one_req,
1513 }
1514 };
1515 
1516 /* OMAP4 has some algs in addition to what OMAP2 has */
1517 static struct ahash_engine_alg algs_sha224_sha256[] = {
1518 {
1519 	.base.init		= omap_sham_init,
1520 	.base.update		= omap_sham_update,
1521 	.base.final		= omap_sham_final,
1522 	.base.finup		= omap_sham_finup,
1523 	.base.digest		= omap_sham_digest,
1524 	.base.halg.digestsize	= SHA224_DIGEST_SIZE,
1525 	.base.halg.base	= {
1526 		.cra_name		= "sha224",
1527 		.cra_driver_name	= "omap-sha224",
1528 		.cra_priority		= 400,
1529 		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1530 						CRYPTO_ALG_ASYNC |
1531 						CRYPTO_ALG_NEED_FALLBACK,
1532 		.cra_blocksize		= SHA224_BLOCK_SIZE,
1533 		.cra_ctxsize		= sizeof(struct omap_sham_ctx),
1534 		.cra_module		= THIS_MODULE,
1535 		.cra_init		= omap_sham_cra_init,
1536 		.cra_exit		= omap_sham_cra_exit,
1537 	},
1538 	.op.do_one_request = omap_sham_hash_one_req,
1539 },
1540 {
1541 	.base.init		= omap_sham_init,
1542 	.base.update		= omap_sham_update,
1543 	.base.final		= omap_sham_final,
1544 	.base.finup		= omap_sham_finup,
1545 	.base.digest		= omap_sham_digest,
1546 	.base.halg.digestsize	= SHA256_DIGEST_SIZE,
1547 	.base.halg.base	= {
1548 		.cra_name		= "sha256",
1549 		.cra_driver_name	= "omap-sha256",
1550 		.cra_priority		= 400,
1551 		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1552 						CRYPTO_ALG_ASYNC |
1553 						CRYPTO_ALG_NEED_FALLBACK,
1554 		.cra_blocksize		= SHA256_BLOCK_SIZE,
1555 		.cra_ctxsize		= sizeof(struct omap_sham_ctx),
1556 		.cra_module		= THIS_MODULE,
1557 		.cra_init		= omap_sham_cra_init,
1558 		.cra_exit		= omap_sham_cra_exit,
1559 	},
1560 	.op.do_one_request = omap_sham_hash_one_req,
1561 },
1562 {
1563 	.base.init		= omap_sham_init,
1564 	.base.update		= omap_sham_update,
1565 	.base.final		= omap_sham_final,
1566 	.base.finup		= omap_sham_finup,
1567 	.base.digest		= omap_sham_digest,
1568 	.base.setkey		= omap_sham_setkey,
1569 	.base.halg.digestsize	= SHA224_DIGEST_SIZE,
1570 	.base.halg.base	= {
1571 		.cra_name		= "hmac(sha224)",
1572 		.cra_driver_name	= "omap-hmac-sha224",
1573 		.cra_priority		= 400,
1574 		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1575 						CRYPTO_ALG_ASYNC |
1576 						CRYPTO_ALG_NEED_FALLBACK,
1577 		.cra_blocksize		= SHA224_BLOCK_SIZE,
1578 		.cra_ctxsize		= sizeof(struct omap_sham_ctx) +
1579 					sizeof(struct omap_sham_hmac_ctx),
1580 		.cra_module		= THIS_MODULE,
1581 		.cra_init		= omap_sham_cra_sha224_init,
1582 		.cra_exit		= omap_sham_cra_exit,
1583 	},
1584 	.op.do_one_request = omap_sham_hash_one_req,
1585 },
1586 {
1587 	.base.init		= omap_sham_init,
1588 	.base.update		= omap_sham_update,
1589 	.base.final		= omap_sham_final,
1590 	.base.finup		= omap_sham_finup,
1591 	.base.digest		= omap_sham_digest,
1592 	.base.setkey		= omap_sham_setkey,
1593 	.base.halg.digestsize	= SHA256_DIGEST_SIZE,
1594 	.base.halg.base	= {
1595 		.cra_name		= "hmac(sha256)",
1596 		.cra_driver_name	= "omap-hmac-sha256",
1597 		.cra_priority		= 400,
1598 		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1599 						CRYPTO_ALG_ASYNC |
1600 						CRYPTO_ALG_NEED_FALLBACK,
1601 		.cra_blocksize		= SHA256_BLOCK_SIZE,
1602 		.cra_ctxsize		= sizeof(struct omap_sham_ctx) +
1603 					sizeof(struct omap_sham_hmac_ctx),
1604 		.cra_module		= THIS_MODULE,
1605 		.cra_init		= omap_sham_cra_sha256_init,
1606 		.cra_exit		= omap_sham_cra_exit,
1607 	},
1608 	.op.do_one_request = omap_sham_hash_one_req,
1609 },
1610 };
1611 
1612 static struct ahash_engine_alg algs_sha384_sha512[] = {
1613 {
1614 	.base.init		= omap_sham_init,
1615 	.base.update		= omap_sham_update,
1616 	.base.final		= omap_sham_final,
1617 	.base.finup		= omap_sham_finup,
1618 	.base.digest		= omap_sham_digest,
1619 	.base.halg.digestsize	= SHA384_DIGEST_SIZE,
1620 	.base.halg.base	= {
1621 		.cra_name		= "sha384",
1622 		.cra_driver_name	= "omap-sha384",
1623 		.cra_priority		= 400,
1624 		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1625 						CRYPTO_ALG_ASYNC |
1626 						CRYPTO_ALG_NEED_FALLBACK,
1627 		.cra_blocksize		= SHA384_BLOCK_SIZE,
1628 		.cra_ctxsize		= sizeof(struct omap_sham_ctx),
1629 		.cra_module		= THIS_MODULE,
1630 		.cra_init		= omap_sham_cra_init,
1631 		.cra_exit		= omap_sham_cra_exit,
1632 	},
1633 	.op.do_one_request = omap_sham_hash_one_req,
1634 },
1635 {
1636 	.base.init		= omap_sham_init,
1637 	.base.update		= omap_sham_update,
1638 	.base.final		= omap_sham_final,
1639 	.base.finup		= omap_sham_finup,
1640 	.base.digest		= omap_sham_digest,
1641 	.base.halg.digestsize	= SHA512_DIGEST_SIZE,
1642 	.base.halg.base	= {
1643 		.cra_name		= "sha512",
1644 		.cra_driver_name	= "omap-sha512",
1645 		.cra_priority		= 400,
1646 		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1647 						CRYPTO_ALG_ASYNC |
1648 						CRYPTO_ALG_NEED_FALLBACK,
1649 		.cra_blocksize		= SHA512_BLOCK_SIZE,
1650 		.cra_ctxsize		= sizeof(struct omap_sham_ctx),
1651 		.cra_module		= THIS_MODULE,
1652 		.cra_init		= omap_sham_cra_init,
1653 		.cra_exit		= omap_sham_cra_exit,
1654 	},
1655 	.op.do_one_request = omap_sham_hash_one_req,
1656 },
1657 {
1658 	.base.init		= omap_sham_init,
1659 	.base.update		= omap_sham_update,
1660 	.base.final		= omap_sham_final,
1661 	.base.finup		= omap_sham_finup,
1662 	.base.digest		= omap_sham_digest,
1663 	.base.setkey		= omap_sham_setkey,
1664 	.base.halg.digestsize	= SHA384_DIGEST_SIZE,
1665 	.base.halg.base	= {
1666 		.cra_name		= "hmac(sha384)",
1667 		.cra_driver_name	= "omap-hmac-sha384",
1668 		.cra_priority		= 400,
1669 		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1670 						CRYPTO_ALG_ASYNC |
1671 						CRYPTO_ALG_NEED_FALLBACK,
1672 		.cra_blocksize		= SHA384_BLOCK_SIZE,
1673 		.cra_ctxsize		= sizeof(struct omap_sham_ctx) +
1674 					sizeof(struct omap_sham_hmac_ctx),
1675 		.cra_module		= THIS_MODULE,
1676 		.cra_init		= omap_sham_cra_sha384_init,
1677 		.cra_exit		= omap_sham_cra_exit,
1678 	},
1679 	.op.do_one_request = omap_sham_hash_one_req,
1680 },
1681 {
1682 	.base.init		= omap_sham_init,
1683 	.base.update		= omap_sham_update,
1684 	.base.final		= omap_sham_final,
1685 	.base.finup		= omap_sham_finup,
1686 	.base.digest		= omap_sham_digest,
1687 	.base.setkey		= omap_sham_setkey,
1688 	.base.halg.digestsize	= SHA512_DIGEST_SIZE,
1689 	.base.halg.base	= {
1690 		.cra_name		= "hmac(sha512)",
1691 		.cra_driver_name	= "omap-hmac-sha512",
1692 		.cra_priority		= 400,
1693 		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1694 						CRYPTO_ALG_ASYNC |
1695 						CRYPTO_ALG_NEED_FALLBACK,
1696 		.cra_blocksize		= SHA512_BLOCK_SIZE,
1697 		.cra_ctxsize		= sizeof(struct omap_sham_ctx) +
1698 					sizeof(struct omap_sham_hmac_ctx),
1699 		.cra_module		= THIS_MODULE,
1700 		.cra_init		= omap_sham_cra_sha512_init,
1701 		.cra_exit		= omap_sham_cra_exit,
1702 	},
1703 	.op.do_one_request = omap_sham_hash_one_req,
1704 },
1705 };
1706 
omap_sham_done_task(unsigned long data)1707 static void omap_sham_done_task(unsigned long data)
1708 {
1709 	struct omap_sham_dev *dd = (struct omap_sham_dev *)data;
1710 	int err = 0;
1711 
1712 	dev_dbg(dd->dev, "%s: flags=%lx\n", __func__, dd->flags);
1713 
1714 	if (test_bit(FLAGS_CPU, &dd->flags)) {
1715 		if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags))
1716 			goto finish;
1717 	} else if (test_bit(FLAGS_DMA_READY, &dd->flags)) {
1718 		if (test_bit(FLAGS_DMA_ACTIVE, &dd->flags)) {
1719 			omap_sham_update_dma_stop(dd);
1720 			if (dd->err) {
1721 				err = dd->err;
1722 				goto finish;
1723 			}
1724 		}
1725 		if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags)) {
1726 			/* hash or semi-hash ready */
1727 			clear_bit(FLAGS_DMA_READY, &dd->flags);
1728 			goto finish;
1729 		}
1730 	}
1731 
1732 	return;
1733 
1734 finish:
1735 	dev_dbg(dd->dev, "update done: err: %d\n", err);
1736 	/* finish curent request */
1737 	omap_sham_finish_req(dd->req, err);
1738 }
1739 
omap_sham_irq_common(struct omap_sham_dev * dd)1740 static irqreturn_t omap_sham_irq_common(struct omap_sham_dev *dd)
1741 {
1742 	set_bit(FLAGS_OUTPUT_READY, &dd->flags);
1743 	tasklet_schedule(&dd->done_task);
1744 
1745 	return IRQ_HANDLED;
1746 }
1747 
omap_sham_irq_omap2(int irq,void * dev_id)1748 static irqreturn_t omap_sham_irq_omap2(int irq, void *dev_id)
1749 {
1750 	struct omap_sham_dev *dd = dev_id;
1751 
1752 	if (unlikely(test_bit(FLAGS_FINAL, &dd->flags)))
1753 		/* final -> allow device to go to power-saving mode */
1754 		omap_sham_write_mask(dd, SHA_REG_CTRL, 0, SHA_REG_CTRL_LENGTH);
1755 
1756 	omap_sham_write_mask(dd, SHA_REG_CTRL, SHA_REG_CTRL_OUTPUT_READY,
1757 				 SHA_REG_CTRL_OUTPUT_READY);
1758 	omap_sham_read(dd, SHA_REG_CTRL);
1759 
1760 	return omap_sham_irq_common(dd);
1761 }
1762 
omap_sham_irq_omap4(int irq,void * dev_id)1763 static irqreturn_t omap_sham_irq_omap4(int irq, void *dev_id)
1764 {
1765 	struct omap_sham_dev *dd = dev_id;
1766 
1767 	omap_sham_write_mask(dd, SHA_REG_MASK(dd), 0, SHA_REG_MASK_IT_EN);
1768 
1769 	return omap_sham_irq_common(dd);
1770 }
1771 
1772 static struct omap_sham_algs_info omap_sham_algs_info_omap2[] = {
1773 	{
1774 		.algs_list	= algs_sha1_md5,
1775 		.size		= ARRAY_SIZE(algs_sha1_md5),
1776 	},
1777 };
1778 
1779 static const struct omap_sham_pdata omap_sham_pdata_omap2 = {
1780 	.algs_info	= omap_sham_algs_info_omap2,
1781 	.algs_info_size	= ARRAY_SIZE(omap_sham_algs_info_omap2),
1782 	.flags		= BIT(FLAGS_BE32_SHA1),
1783 	.digest_size	= SHA1_DIGEST_SIZE,
1784 	.copy_hash	= omap_sham_copy_hash_omap2,
1785 	.write_ctrl	= omap_sham_write_ctrl_omap2,
1786 	.trigger	= omap_sham_trigger_omap2,
1787 	.poll_irq	= omap_sham_poll_irq_omap2,
1788 	.intr_hdlr	= omap_sham_irq_omap2,
1789 	.idigest_ofs	= 0x00,
1790 	.din_ofs	= 0x1c,
1791 	.digcnt_ofs	= 0x14,
1792 	.rev_ofs	= 0x5c,
1793 	.mask_ofs	= 0x60,
1794 	.sysstatus_ofs	= 0x64,
1795 	.major_mask	= 0xf0,
1796 	.major_shift	= 4,
1797 	.minor_mask	= 0x0f,
1798 	.minor_shift	= 0,
1799 };
1800 
1801 #ifdef CONFIG_OF
1802 static struct omap_sham_algs_info omap_sham_algs_info_omap4[] = {
1803 	{
1804 		.algs_list	= algs_sha1_md5,
1805 		.size		= ARRAY_SIZE(algs_sha1_md5),
1806 	},
1807 	{
1808 		.algs_list	= algs_sha224_sha256,
1809 		.size		= ARRAY_SIZE(algs_sha224_sha256),
1810 	},
1811 };
1812 
1813 static const struct omap_sham_pdata omap_sham_pdata_omap4 = {
1814 	.algs_info	= omap_sham_algs_info_omap4,
1815 	.algs_info_size	= ARRAY_SIZE(omap_sham_algs_info_omap4),
1816 	.flags		= BIT(FLAGS_AUTO_XOR),
1817 	.digest_size	= SHA256_DIGEST_SIZE,
1818 	.copy_hash	= omap_sham_copy_hash_omap4,
1819 	.write_ctrl	= omap_sham_write_ctrl_omap4,
1820 	.trigger	= omap_sham_trigger_omap4,
1821 	.poll_irq	= omap_sham_poll_irq_omap4,
1822 	.intr_hdlr	= omap_sham_irq_omap4,
1823 	.idigest_ofs	= 0x020,
1824 	.odigest_ofs	= 0x0,
1825 	.din_ofs	= 0x080,
1826 	.digcnt_ofs	= 0x040,
1827 	.rev_ofs	= 0x100,
1828 	.mask_ofs	= 0x110,
1829 	.sysstatus_ofs	= 0x114,
1830 	.mode_ofs	= 0x44,
1831 	.length_ofs	= 0x48,
1832 	.major_mask	= 0x0700,
1833 	.major_shift	= 8,
1834 	.minor_mask	= 0x003f,
1835 	.minor_shift	= 0,
1836 };
1837 
1838 static struct omap_sham_algs_info omap_sham_algs_info_omap5[] = {
1839 	{
1840 		.algs_list	= algs_sha1_md5,
1841 		.size		= ARRAY_SIZE(algs_sha1_md5),
1842 	},
1843 	{
1844 		.algs_list	= algs_sha224_sha256,
1845 		.size		= ARRAY_SIZE(algs_sha224_sha256),
1846 	},
1847 	{
1848 		.algs_list	= algs_sha384_sha512,
1849 		.size		= ARRAY_SIZE(algs_sha384_sha512),
1850 	},
1851 };
1852 
1853 static const struct omap_sham_pdata omap_sham_pdata_omap5 = {
1854 	.algs_info	= omap_sham_algs_info_omap5,
1855 	.algs_info_size	= ARRAY_SIZE(omap_sham_algs_info_omap5),
1856 	.flags		= BIT(FLAGS_AUTO_XOR),
1857 	.digest_size	= SHA512_DIGEST_SIZE,
1858 	.copy_hash	= omap_sham_copy_hash_omap4,
1859 	.write_ctrl	= omap_sham_write_ctrl_omap4,
1860 	.trigger	= omap_sham_trigger_omap4,
1861 	.poll_irq	= omap_sham_poll_irq_omap4,
1862 	.intr_hdlr	= omap_sham_irq_omap4,
1863 	.idigest_ofs	= 0x240,
1864 	.odigest_ofs	= 0x200,
1865 	.din_ofs	= 0x080,
1866 	.digcnt_ofs	= 0x280,
1867 	.rev_ofs	= 0x100,
1868 	.mask_ofs	= 0x110,
1869 	.sysstatus_ofs	= 0x114,
1870 	.mode_ofs	= 0x284,
1871 	.length_ofs	= 0x288,
1872 	.major_mask	= 0x0700,
1873 	.major_shift	= 8,
1874 	.minor_mask	= 0x003f,
1875 	.minor_shift	= 0,
1876 };
1877 
1878 static const struct of_device_id omap_sham_of_match[] = {
1879 	{
1880 		.compatible	= "ti,omap2-sham",
1881 		.data		= &omap_sham_pdata_omap2,
1882 	},
1883 	{
1884 		.compatible	= "ti,omap3-sham",
1885 		.data		= &omap_sham_pdata_omap2,
1886 	},
1887 	{
1888 		.compatible	= "ti,omap4-sham",
1889 		.data		= &omap_sham_pdata_omap4,
1890 	},
1891 	{
1892 		.compatible	= "ti,omap5-sham",
1893 		.data		= &omap_sham_pdata_omap5,
1894 	},
1895 	{},
1896 };
1897 MODULE_DEVICE_TABLE(of, omap_sham_of_match);
1898 
omap_sham_get_res_of(struct omap_sham_dev * dd,struct device * dev,struct resource * res)1899 static int omap_sham_get_res_of(struct omap_sham_dev *dd,
1900 		struct device *dev, struct resource *res)
1901 {
1902 	struct device_node *node = dev->of_node;
1903 	int err = 0;
1904 
1905 	dd->pdata = of_device_get_match_data(dev);
1906 	if (!dd->pdata) {
1907 		dev_err(dev, "no compatible OF match\n");
1908 		err = -EINVAL;
1909 		goto err;
1910 	}
1911 
1912 	err = of_address_to_resource(node, 0, res);
1913 	if (err < 0) {
1914 		dev_err(dev, "can't translate OF node address\n");
1915 		err = -EINVAL;
1916 		goto err;
1917 	}
1918 
1919 	dd->irq = irq_of_parse_and_map(node, 0);
1920 	if (!dd->irq) {
1921 		dev_err(dev, "can't translate OF irq value\n");
1922 		err = -EINVAL;
1923 		goto err;
1924 	}
1925 
1926 err:
1927 	return err;
1928 }
1929 #else
1930 static const struct of_device_id omap_sham_of_match[] = {
1931 	{},
1932 };
1933 
omap_sham_get_res_of(struct omap_sham_dev * dd,struct device * dev,struct resource * res)1934 static int omap_sham_get_res_of(struct omap_sham_dev *dd,
1935 		struct device *dev, struct resource *res)
1936 {
1937 	return -EINVAL;
1938 }
1939 #endif
1940 
omap_sham_get_res_pdev(struct omap_sham_dev * dd,struct platform_device * pdev,struct resource * res)1941 static int omap_sham_get_res_pdev(struct omap_sham_dev *dd,
1942 		struct platform_device *pdev, struct resource *res)
1943 {
1944 	struct device *dev = &pdev->dev;
1945 	struct resource *r;
1946 	int err = 0;
1947 
1948 	/* Get the base address */
1949 	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1950 	if (!r) {
1951 		dev_err(dev, "no MEM resource info\n");
1952 		err = -ENODEV;
1953 		goto err;
1954 	}
1955 	memcpy(res, r, sizeof(*res));
1956 
1957 	/* Get the IRQ */
1958 	dd->irq = platform_get_irq(pdev, 0);
1959 	if (dd->irq < 0) {
1960 		err = dd->irq;
1961 		goto err;
1962 	}
1963 
1964 	/* Only OMAP2/3 can be non-DT */
1965 	dd->pdata = &omap_sham_pdata_omap2;
1966 
1967 err:
1968 	return err;
1969 }
1970 
fallback_show(struct device * dev,struct device_attribute * attr,char * buf)1971 static ssize_t fallback_show(struct device *dev, struct device_attribute *attr,
1972 			     char *buf)
1973 {
1974 	struct omap_sham_dev *dd = dev_get_drvdata(dev);
1975 
1976 	return sprintf(buf, "%d\n", dd->fallback_sz);
1977 }
1978 
fallback_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t size)1979 static ssize_t fallback_store(struct device *dev, struct device_attribute *attr,
1980 			      const char *buf, size_t size)
1981 {
1982 	struct omap_sham_dev *dd = dev_get_drvdata(dev);
1983 	ssize_t status;
1984 	long value;
1985 
1986 	status = kstrtol(buf, 0, &value);
1987 	if (status)
1988 		return status;
1989 
1990 	/* HW accelerator only works with buffers > 9 */
1991 	if (value < 9) {
1992 		dev_err(dev, "minimum fallback size 9\n");
1993 		return -EINVAL;
1994 	}
1995 
1996 	dd->fallback_sz = value;
1997 
1998 	return size;
1999 }
2000 
queue_len_show(struct device * dev,struct device_attribute * attr,char * buf)2001 static ssize_t queue_len_show(struct device *dev, struct device_attribute *attr,
2002 			      char *buf)
2003 {
2004 	struct omap_sham_dev *dd = dev_get_drvdata(dev);
2005 
2006 	return sprintf(buf, "%d\n", dd->queue.max_qlen);
2007 }
2008 
queue_len_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t size)2009 static ssize_t queue_len_store(struct device *dev,
2010 			       struct device_attribute *attr, const char *buf,
2011 			       size_t size)
2012 {
2013 	struct omap_sham_dev *dd = dev_get_drvdata(dev);
2014 	ssize_t status;
2015 	long value;
2016 
2017 	status = kstrtol(buf, 0, &value);
2018 	if (status)
2019 		return status;
2020 
2021 	if (value < 1)
2022 		return -EINVAL;
2023 
2024 	/*
2025 	 * Changing the queue size in fly is safe, if size becomes smaller
2026 	 * than current size, it will just not accept new entries until
2027 	 * it has shrank enough.
2028 	 */
2029 	dd->queue.max_qlen = value;
2030 
2031 	return size;
2032 }
2033 
2034 static DEVICE_ATTR_RW(queue_len);
2035 static DEVICE_ATTR_RW(fallback);
2036 
2037 static struct attribute *omap_sham_attrs[] = {
2038 	&dev_attr_queue_len.attr,
2039 	&dev_attr_fallback.attr,
2040 	NULL,
2041 };
2042 
2043 static const struct attribute_group omap_sham_attr_group = {
2044 	.attrs = omap_sham_attrs,
2045 };
2046 
omap_sham_probe(struct platform_device * pdev)2047 static int omap_sham_probe(struct platform_device *pdev)
2048 {
2049 	struct omap_sham_dev *dd;
2050 	struct device *dev = &pdev->dev;
2051 	struct resource res;
2052 	dma_cap_mask_t mask;
2053 	int err, i, j;
2054 	u32 rev;
2055 
2056 	dd = devm_kzalloc(dev, sizeof(struct omap_sham_dev), GFP_KERNEL);
2057 	if (dd == NULL) {
2058 		dev_err(dev, "unable to alloc data struct.\n");
2059 		err = -ENOMEM;
2060 		goto data_err;
2061 	}
2062 	dd->dev = dev;
2063 	platform_set_drvdata(pdev, dd);
2064 
2065 	INIT_LIST_HEAD(&dd->list);
2066 	tasklet_init(&dd->done_task, omap_sham_done_task, (unsigned long)dd);
2067 	crypto_init_queue(&dd->queue, OMAP_SHAM_QUEUE_LENGTH);
2068 
2069 	err = (dev->of_node) ? omap_sham_get_res_of(dd, dev, &res) :
2070 			       omap_sham_get_res_pdev(dd, pdev, &res);
2071 	if (err)
2072 		goto data_err;
2073 
2074 	dd->io_base = devm_ioremap_resource(dev, &res);
2075 	if (IS_ERR(dd->io_base)) {
2076 		err = PTR_ERR(dd->io_base);
2077 		goto data_err;
2078 	}
2079 	dd->phys_base = res.start;
2080 
2081 	err = devm_request_irq(dev, dd->irq, dd->pdata->intr_hdlr,
2082 			       IRQF_TRIGGER_NONE, dev_name(dev), dd);
2083 	if (err) {
2084 		dev_err(dev, "unable to request irq %d, err = %d\n",
2085 			dd->irq, err);
2086 		goto data_err;
2087 	}
2088 
2089 	dma_cap_zero(mask);
2090 	dma_cap_set(DMA_SLAVE, mask);
2091 
2092 	dd->dma_lch = dma_request_chan(dev, "rx");
2093 	if (IS_ERR(dd->dma_lch)) {
2094 		err = PTR_ERR(dd->dma_lch);
2095 		if (err == -EPROBE_DEFER)
2096 			goto data_err;
2097 
2098 		dd->polling_mode = 1;
2099 		dev_dbg(dev, "using polling mode instead of dma\n");
2100 	}
2101 
2102 	dd->flags |= dd->pdata->flags;
2103 	sham.flags |= dd->pdata->flags;
2104 
2105 	pm_runtime_use_autosuspend(dev);
2106 	pm_runtime_set_autosuspend_delay(dev, DEFAULT_AUTOSUSPEND_DELAY);
2107 
2108 	dd->fallback_sz = OMAP_SHA_DMA_THRESHOLD;
2109 
2110 	pm_runtime_enable(dev);
2111 
2112 	err = pm_runtime_resume_and_get(dev);
2113 	if (err < 0) {
2114 		dev_err(dev, "failed to get sync: %d\n", err);
2115 		goto err_pm;
2116 	}
2117 
2118 	rev = omap_sham_read(dd, SHA_REG_REV(dd));
2119 	pm_runtime_put_sync(&pdev->dev);
2120 
2121 	dev_info(dev, "hw accel on OMAP rev %u.%u\n",
2122 		(rev & dd->pdata->major_mask) >> dd->pdata->major_shift,
2123 		(rev & dd->pdata->minor_mask) >> dd->pdata->minor_shift);
2124 
2125 	spin_lock_bh(&sham.lock);
2126 	list_add_tail(&dd->list, &sham.dev_list);
2127 	spin_unlock_bh(&sham.lock);
2128 
2129 	dd->engine = crypto_engine_alloc_init(dev, 1);
2130 	if (!dd->engine) {
2131 		err = -ENOMEM;
2132 		goto err_engine;
2133 	}
2134 
2135 	err = crypto_engine_start(dd->engine);
2136 	if (err)
2137 		goto err_engine_start;
2138 
2139 	for (i = 0; i < dd->pdata->algs_info_size; i++) {
2140 		if (dd->pdata->algs_info[i].registered)
2141 			break;
2142 
2143 		for (j = 0; j < dd->pdata->algs_info[i].size; j++) {
2144 			struct ahash_engine_alg *ealg;
2145 			struct ahash_alg *alg;
2146 
2147 			ealg = &dd->pdata->algs_info[i].algs_list[j];
2148 			alg = &ealg->base;
2149 			alg->export = omap_sham_export;
2150 			alg->import = omap_sham_import;
2151 			alg->halg.statesize = sizeof(struct omap_sham_reqctx) +
2152 					      BUFLEN;
2153 			err = crypto_engine_register_ahash(ealg);
2154 			if (err)
2155 				goto err_algs;
2156 
2157 			dd->pdata->algs_info[i].registered++;
2158 		}
2159 	}
2160 
2161 	err = sysfs_create_group(&dev->kobj, &omap_sham_attr_group);
2162 	if (err) {
2163 		dev_err(dev, "could not create sysfs device attrs\n");
2164 		goto err_algs;
2165 	}
2166 
2167 	return 0;
2168 
2169 err_algs:
2170 	for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
2171 		for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
2172 			crypto_engine_unregister_ahash(
2173 					&dd->pdata->algs_info[i].algs_list[j]);
2174 err_engine_start:
2175 	crypto_engine_exit(dd->engine);
2176 err_engine:
2177 	spin_lock_bh(&sham.lock);
2178 	list_del(&dd->list);
2179 	spin_unlock_bh(&sham.lock);
2180 err_pm:
2181 	pm_runtime_dont_use_autosuspend(dev);
2182 	pm_runtime_disable(dev);
2183 	if (!dd->polling_mode)
2184 		dma_release_channel(dd->dma_lch);
2185 data_err:
2186 	dev_err(dev, "initialization failed.\n");
2187 
2188 	return err;
2189 }
2190 
omap_sham_remove(struct platform_device * pdev)2191 static void omap_sham_remove(struct platform_device *pdev)
2192 {
2193 	struct omap_sham_dev *dd;
2194 	int i, j;
2195 
2196 	dd = platform_get_drvdata(pdev);
2197 
2198 	spin_lock_bh(&sham.lock);
2199 	list_del(&dd->list);
2200 	spin_unlock_bh(&sham.lock);
2201 	for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
2202 		for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) {
2203 			crypto_engine_unregister_ahash(
2204 					&dd->pdata->algs_info[i].algs_list[j]);
2205 			dd->pdata->algs_info[i].registered--;
2206 		}
2207 	tasklet_kill(&dd->done_task);
2208 	pm_runtime_dont_use_autosuspend(&pdev->dev);
2209 	pm_runtime_disable(&pdev->dev);
2210 
2211 	if (!dd->polling_mode)
2212 		dma_release_channel(dd->dma_lch);
2213 
2214 	sysfs_remove_group(&dd->dev->kobj, &omap_sham_attr_group);
2215 }
2216 
2217 static struct platform_driver omap_sham_driver = {
2218 	.probe	= omap_sham_probe,
2219 	.remove_new = omap_sham_remove,
2220 	.driver	= {
2221 		.name	= "omap-sham",
2222 		.of_match_table	= omap_sham_of_match,
2223 	},
2224 };
2225 
2226 module_platform_driver(omap_sham_driver);
2227 
2228 MODULE_DESCRIPTION("OMAP SHA1/MD5 hw acceleration support.");
2229 MODULE_LICENSE("GPL v2");
2230 MODULE_AUTHOR("Dmitry Kasatkin");
2231 MODULE_ALIAS("platform:omap-sham");
2232