xref: /linux/drivers/crypto/bcm/spu2.c (revision 621cde16e49b3ecf7d59a8106a20aaebfb4a59a9)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright 2016 Broadcom
4  */
5 
6 /*
7  * This file works with the SPU2 version of the SPU. SPU2 has different message
8  * formats than the previous version of the SPU. All SPU message format
9  * differences should be hidden in the spux.c,h files.
10  */
11 
12 #include <linux/kernel.h>
13 #include <linux/string.h>
14 
15 #include "util.h"
16 #include "spu.h"
17 #include "spu2.h"
18 
19 #define SPU2_TX_STATUS_LEN  0	/* SPU2 has no STATUS in input packet */
20 
21 /*
22  * Controlled by pkt_stat_cnt field in CRYPTO_SS_SPU0_CORE_SPU2_CONTROL0
23  * register. Defaults to 2.
24  */
25 #define SPU2_RX_STATUS_LEN  2
26 
27 enum spu2_proto_sel {
28 	SPU2_PROTO_RESV = 0,
29 	SPU2_MACSEC_SECTAG8_ECB = 1,
30 	SPU2_MACSEC_SECTAG8_SCB = 2,
31 	SPU2_MACSEC_SECTAG16 = 3,
32 	SPU2_MACSEC_SECTAG16_8_XPN = 4,
33 	SPU2_IPSEC = 5,
34 	SPU2_IPSEC_ESN = 6,
35 	SPU2_TLS_CIPHER = 7,
36 	SPU2_TLS_AEAD = 8,
37 	SPU2_DTLS_CIPHER = 9,
38 	SPU2_DTLS_AEAD = 10
39 };
40 
41 static char *spu2_cipher_type_names[] = { "None", "AES128", "AES192", "AES256",
42 	"DES", "3DES"
43 };
44 
45 static char *spu2_cipher_mode_names[] = { "ECB", "CBC", "CTR", "CFB", "OFB",
46 	"XTS", "CCM", "GCM"
47 };
48 
49 static char *spu2_hash_type_names[] = { "None", "AES128", "AES192", "AES256",
50 	"Reserved", "Reserved", "MD5", "SHA1", "SHA224", "SHA256", "SHA384",
51 	"SHA512", "SHA512/224", "SHA512/256", "SHA3-224", "SHA3-256",
52 	"SHA3-384", "SHA3-512"
53 };
54 
55 static char *spu2_hash_mode_names[] = { "CMAC", "CBC-MAC", "XCBC-MAC", "HMAC",
56 	"Rabin", "CCM", "GCM", "Reserved"
57 };
58 
spu2_ciph_type_name(enum spu2_cipher_type cipher_type)59 static char *spu2_ciph_type_name(enum spu2_cipher_type cipher_type)
60 {
61 	if (cipher_type >= SPU2_CIPHER_TYPE_LAST)
62 		return "Reserved";
63 	return spu2_cipher_type_names[cipher_type];
64 }
65 
spu2_ciph_mode_name(enum spu2_cipher_mode cipher_mode)66 static char *spu2_ciph_mode_name(enum spu2_cipher_mode cipher_mode)
67 {
68 	if (cipher_mode >= SPU2_CIPHER_MODE_LAST)
69 		return "Reserved";
70 	return spu2_cipher_mode_names[cipher_mode];
71 }
72 
spu2_hash_type_name(enum spu2_hash_type hash_type)73 static char *spu2_hash_type_name(enum spu2_hash_type hash_type)
74 {
75 	if (hash_type >= SPU2_HASH_TYPE_LAST)
76 		return "Reserved";
77 	return spu2_hash_type_names[hash_type];
78 }
79 
spu2_hash_mode_name(enum spu2_hash_mode hash_mode)80 static char *spu2_hash_mode_name(enum spu2_hash_mode hash_mode)
81 {
82 	if (hash_mode >= SPU2_HASH_MODE_LAST)
83 		return "Reserved";
84 	return spu2_hash_mode_names[hash_mode];
85 }
86 
87 /*
88  * Convert from a software cipher mode value to the corresponding value
89  * for SPU2.
90  */
spu2_cipher_mode_xlate(enum spu_cipher_mode cipher_mode,enum spu2_cipher_mode * spu2_mode)91 static int spu2_cipher_mode_xlate(enum spu_cipher_mode cipher_mode,
92 				  enum spu2_cipher_mode *spu2_mode)
93 {
94 	switch (cipher_mode) {
95 	case CIPHER_MODE_ECB:
96 		*spu2_mode = SPU2_CIPHER_MODE_ECB;
97 		break;
98 	case CIPHER_MODE_CBC:
99 		*spu2_mode = SPU2_CIPHER_MODE_CBC;
100 		break;
101 	case CIPHER_MODE_OFB:
102 		*spu2_mode = SPU2_CIPHER_MODE_OFB;
103 		break;
104 	case CIPHER_MODE_CFB:
105 		*spu2_mode = SPU2_CIPHER_MODE_CFB;
106 		break;
107 	case CIPHER_MODE_CTR:
108 		*spu2_mode = SPU2_CIPHER_MODE_CTR;
109 		break;
110 	case CIPHER_MODE_CCM:
111 		*spu2_mode = SPU2_CIPHER_MODE_CCM;
112 		break;
113 	case CIPHER_MODE_GCM:
114 		*spu2_mode = SPU2_CIPHER_MODE_GCM;
115 		break;
116 	case CIPHER_MODE_XTS:
117 		*spu2_mode = SPU2_CIPHER_MODE_XTS;
118 		break;
119 	default:
120 		return -EINVAL;
121 	}
122 	return 0;
123 }
124 
125 /**
126  * spu2_cipher_xlate() - Convert a cipher {alg/mode/type} triple to a SPU2
127  * cipher type and mode.
128  * @cipher_alg:  [in]  cipher algorithm value from software enumeration
129  * @cipher_mode: [in]  cipher mode value from software enumeration
130  * @cipher_type: [in]  cipher type value from software enumeration
131  * @spu2_type:   [out] cipher type value used by spu2 hardware
132  * @spu2_mode:   [out] cipher mode value used by spu2 hardware
133  *
134  * Return:  0 if successful
135  */
spu2_cipher_xlate(enum spu_cipher_alg cipher_alg,enum spu_cipher_mode cipher_mode,enum spu_cipher_type cipher_type,enum spu2_cipher_type * spu2_type,enum spu2_cipher_mode * spu2_mode)136 static int spu2_cipher_xlate(enum spu_cipher_alg cipher_alg,
137 			     enum spu_cipher_mode cipher_mode,
138 			     enum spu_cipher_type cipher_type,
139 			     enum spu2_cipher_type *spu2_type,
140 			     enum spu2_cipher_mode *spu2_mode)
141 {
142 	int err;
143 
144 	err = spu2_cipher_mode_xlate(cipher_mode, spu2_mode);
145 	if (err) {
146 		flow_log("Invalid cipher mode %d\n", cipher_mode);
147 		return err;
148 	}
149 
150 	switch (cipher_alg) {
151 	case CIPHER_ALG_NONE:
152 		*spu2_type = SPU2_CIPHER_TYPE_NONE;
153 		break;
154 	case CIPHER_ALG_RC4:
155 		/* SPU2 does not support RC4 */
156 		err = -EINVAL;
157 		*spu2_type = SPU2_CIPHER_TYPE_NONE;
158 		break;
159 	case CIPHER_ALG_DES:
160 		*spu2_type = SPU2_CIPHER_TYPE_DES;
161 		break;
162 	case CIPHER_ALG_3DES:
163 		*spu2_type = SPU2_CIPHER_TYPE_3DES;
164 		break;
165 	case CIPHER_ALG_AES:
166 		switch (cipher_type) {
167 		case CIPHER_TYPE_AES128:
168 			*spu2_type = SPU2_CIPHER_TYPE_AES128;
169 			break;
170 		case CIPHER_TYPE_AES192:
171 			*spu2_type = SPU2_CIPHER_TYPE_AES192;
172 			break;
173 		case CIPHER_TYPE_AES256:
174 			*spu2_type = SPU2_CIPHER_TYPE_AES256;
175 			break;
176 		default:
177 			err = -EINVAL;
178 		}
179 		break;
180 	case CIPHER_ALG_LAST:
181 	default:
182 		err = -EINVAL;
183 		break;
184 	}
185 
186 	if (err)
187 		flow_log("Invalid cipher alg %d or type %d\n",
188 			 cipher_alg, cipher_type);
189 	return err;
190 }
191 
192 /*
193  * Convert from a software hash mode value to the corresponding value
194  * for SPU2. Note that HASH_MODE_NONE and HASH_MODE_XCBC have the same value.
195  */
spu2_hash_mode_xlate(enum hash_mode hash_mode,enum spu2_hash_mode * spu2_mode)196 static int spu2_hash_mode_xlate(enum hash_mode hash_mode,
197 				enum spu2_hash_mode *spu2_mode)
198 {
199 	switch (hash_mode) {
200 	case HASH_MODE_XCBC:
201 		*spu2_mode = SPU2_HASH_MODE_XCBC_MAC;
202 		break;
203 	case HASH_MODE_CMAC:
204 		*spu2_mode = SPU2_HASH_MODE_CMAC;
205 		break;
206 	case HASH_MODE_HMAC:
207 		*spu2_mode = SPU2_HASH_MODE_HMAC;
208 		break;
209 	case HASH_MODE_CCM:
210 		*spu2_mode = SPU2_HASH_MODE_CCM;
211 		break;
212 	case HASH_MODE_GCM:
213 		*spu2_mode = SPU2_HASH_MODE_GCM;
214 		break;
215 	default:
216 		return -EINVAL;
217 	}
218 	return 0;
219 }
220 
221 /**
222  * spu2_hash_xlate() - Convert a hash {alg/mode/type} triple to a SPU2 hash type
223  * and mode.
224  * @hash_alg:  [in] hash algorithm value from software enumeration
225  * @hash_mode: [in] hash mode value from software enumeration
226  * @hash_type: [in] hash type value from software enumeration
227  * @ciph_type: [in] cipher type value from software enumeration
228  * @spu2_type: [out] hash type value used by SPU2 hardware
229  * @spu2_mode: [out] hash mode value used by SPU2 hardware
230  *
231  * Return:  0 if successful
232  */
233 static int
spu2_hash_xlate(enum hash_alg hash_alg,enum hash_mode hash_mode,enum hash_type hash_type,enum spu_cipher_type ciph_type,enum spu2_hash_type * spu2_type,enum spu2_hash_mode * spu2_mode)234 spu2_hash_xlate(enum hash_alg hash_alg, enum hash_mode hash_mode,
235 		enum hash_type hash_type, enum spu_cipher_type ciph_type,
236 		enum spu2_hash_type *spu2_type, enum spu2_hash_mode *spu2_mode)
237 {
238 	int err;
239 
240 	err = spu2_hash_mode_xlate(hash_mode, spu2_mode);
241 	if (err) {
242 		flow_log("Invalid hash mode %d\n", hash_mode);
243 		return err;
244 	}
245 
246 	switch (hash_alg) {
247 	case HASH_ALG_NONE:
248 		*spu2_type = SPU2_HASH_TYPE_NONE;
249 		break;
250 	case HASH_ALG_MD5:
251 		*spu2_type = SPU2_HASH_TYPE_MD5;
252 		break;
253 	case HASH_ALG_SHA1:
254 		*spu2_type = SPU2_HASH_TYPE_SHA1;
255 		break;
256 	case HASH_ALG_SHA224:
257 		*spu2_type = SPU2_HASH_TYPE_SHA224;
258 		break;
259 	case HASH_ALG_SHA256:
260 		*spu2_type = SPU2_HASH_TYPE_SHA256;
261 		break;
262 	case HASH_ALG_SHA384:
263 		*spu2_type = SPU2_HASH_TYPE_SHA384;
264 		break;
265 	case HASH_ALG_SHA512:
266 		*spu2_type = SPU2_HASH_TYPE_SHA512;
267 		break;
268 	case HASH_ALG_AES:
269 		switch (ciph_type) {
270 		case CIPHER_TYPE_AES128:
271 			*spu2_type = SPU2_HASH_TYPE_AES128;
272 			break;
273 		case CIPHER_TYPE_AES192:
274 			*spu2_type = SPU2_HASH_TYPE_AES192;
275 			break;
276 		case CIPHER_TYPE_AES256:
277 			*spu2_type = SPU2_HASH_TYPE_AES256;
278 			break;
279 		default:
280 			err = -EINVAL;
281 		}
282 		break;
283 	case HASH_ALG_SHA3_224:
284 		*spu2_type = SPU2_HASH_TYPE_SHA3_224;
285 		break;
286 	case HASH_ALG_SHA3_256:
287 		*spu2_type = SPU2_HASH_TYPE_SHA3_256;
288 		break;
289 	case HASH_ALG_SHA3_384:
290 		*spu2_type = SPU2_HASH_TYPE_SHA3_384;
291 		break;
292 	case HASH_ALG_SHA3_512:
293 		*spu2_type = SPU2_HASH_TYPE_SHA3_512;
294 		break;
295 	case HASH_ALG_LAST:
296 	default:
297 		err = -EINVAL;
298 		break;
299 	}
300 
301 	if (err)
302 		flow_log("Invalid hash alg %d or type %d\n",
303 			 hash_alg, hash_type);
304 	return err;
305 }
306 
307 /* Dump FMD ctrl0. The ctrl0 input is in host byte order */
spu2_dump_fmd_ctrl0(u64 ctrl0)308 static void spu2_dump_fmd_ctrl0(u64 ctrl0)
309 {
310 	enum spu2_cipher_type ciph_type;
311 	enum spu2_cipher_mode ciph_mode;
312 	enum spu2_hash_type hash_type;
313 	enum spu2_hash_mode hash_mode;
314 	char *ciph_name;
315 	char *ciph_mode_name;
316 	char *hash_name;
317 	char *hash_mode_name;
318 	u8 cfb;
319 	u8 proto;
320 
321 	packet_log(" FMD CTRL0 %#16llx\n", ctrl0);
322 	if (ctrl0 & SPU2_CIPH_ENCRYPT_EN)
323 		packet_log("  encrypt\n");
324 	else
325 		packet_log("  decrypt\n");
326 
327 	ciph_type = (ctrl0 & SPU2_CIPH_TYPE) >> SPU2_CIPH_TYPE_SHIFT;
328 	ciph_name = spu2_ciph_type_name(ciph_type);
329 	packet_log("  Cipher type: %s\n", ciph_name);
330 
331 	if (ciph_type != SPU2_CIPHER_TYPE_NONE) {
332 		ciph_mode = (ctrl0 & SPU2_CIPH_MODE) >> SPU2_CIPH_MODE_SHIFT;
333 		ciph_mode_name = spu2_ciph_mode_name(ciph_mode);
334 		packet_log("  Cipher mode: %s\n", ciph_mode_name);
335 	}
336 
337 	cfb = (ctrl0 & SPU2_CFB_MASK) >> SPU2_CFB_MASK_SHIFT;
338 	packet_log("  CFB %#x\n", cfb);
339 
340 	proto = (ctrl0 & SPU2_PROTO_SEL) >> SPU2_PROTO_SEL_SHIFT;
341 	packet_log("  protocol %#x\n", proto);
342 
343 	if (ctrl0 & SPU2_HASH_FIRST)
344 		packet_log("  hash first\n");
345 	else
346 		packet_log("  cipher first\n");
347 
348 	if (ctrl0 & SPU2_CHK_TAG)
349 		packet_log("  check tag\n");
350 
351 	hash_type = (ctrl0 & SPU2_HASH_TYPE) >> SPU2_HASH_TYPE_SHIFT;
352 	hash_name = spu2_hash_type_name(hash_type);
353 	packet_log("  Hash type: %s\n", hash_name);
354 
355 	if (hash_type != SPU2_HASH_TYPE_NONE) {
356 		hash_mode = (ctrl0 & SPU2_HASH_MODE) >> SPU2_HASH_MODE_SHIFT;
357 		hash_mode_name = spu2_hash_mode_name(hash_mode);
358 		packet_log("  Hash mode: %s\n", hash_mode_name);
359 	}
360 
361 	if (ctrl0 & SPU2_CIPH_PAD_EN) {
362 		packet_log("  Cipher pad: %#2llx\n",
363 			   (ctrl0 & SPU2_CIPH_PAD) >> SPU2_CIPH_PAD_SHIFT);
364 	}
365 }
366 
367 /* Dump FMD ctrl1. The ctrl1 input is in host byte order */
spu2_dump_fmd_ctrl1(u64 ctrl1)368 static void spu2_dump_fmd_ctrl1(u64 ctrl1)
369 {
370 	u8 hash_key_len;
371 	u8 ciph_key_len;
372 	u8 ret_iv_len;
373 	u8 iv_offset;
374 	u8 iv_len;
375 	u8 hash_tag_len;
376 	u8 ret_md;
377 
378 	packet_log(" FMD CTRL1 %#16llx\n", ctrl1);
379 	if (ctrl1 & SPU2_TAG_LOC)
380 		packet_log("  Tag after payload\n");
381 
382 	packet_log("  Msg includes ");
383 	if (ctrl1 & SPU2_HAS_FR_DATA)
384 		packet_log("FD ");
385 	if (ctrl1 & SPU2_HAS_AAD1)
386 		packet_log("AAD1 ");
387 	if (ctrl1 & SPU2_HAS_NAAD)
388 		packet_log("NAAD ");
389 	if (ctrl1 & SPU2_HAS_AAD2)
390 		packet_log("AAD2 ");
391 	if (ctrl1 & SPU2_HAS_ESN)
392 		packet_log("ESN ");
393 	packet_log("\n");
394 
395 	hash_key_len = (ctrl1 & SPU2_HASH_KEY_LEN) >> SPU2_HASH_KEY_LEN_SHIFT;
396 	packet_log("  Hash key len %u\n", hash_key_len);
397 
398 	ciph_key_len = (ctrl1 & SPU2_CIPH_KEY_LEN) >> SPU2_CIPH_KEY_LEN_SHIFT;
399 	packet_log("  Cipher key len %u\n", ciph_key_len);
400 
401 	if (ctrl1 & SPU2_GENIV)
402 		packet_log("  Generate IV\n");
403 
404 	if (ctrl1 & SPU2_HASH_IV)
405 		packet_log("  IV included in hash\n");
406 
407 	if (ctrl1 & SPU2_RET_IV)
408 		packet_log("  Return IV in output before payload\n");
409 
410 	ret_iv_len = (ctrl1 & SPU2_RET_IV_LEN) >> SPU2_RET_IV_LEN_SHIFT;
411 	packet_log("  Length of returned IV %u bytes\n",
412 		   ret_iv_len ? ret_iv_len : 16);
413 
414 	iv_offset = (ctrl1 & SPU2_IV_OFFSET) >> SPU2_IV_OFFSET_SHIFT;
415 	packet_log("  IV offset %u\n", iv_offset);
416 
417 	iv_len = (ctrl1 & SPU2_IV_LEN) >> SPU2_IV_LEN_SHIFT;
418 	packet_log("  Input IV len %u bytes\n", iv_len);
419 
420 	hash_tag_len = (ctrl1 & SPU2_HASH_TAG_LEN) >> SPU2_HASH_TAG_LEN_SHIFT;
421 	packet_log("  Hash tag length %u bytes\n", hash_tag_len);
422 
423 	packet_log("  Return ");
424 	ret_md = (ctrl1 & SPU2_RETURN_MD) >> SPU2_RETURN_MD_SHIFT;
425 	if (ret_md)
426 		packet_log("FMD ");
427 	if (ret_md == SPU2_RET_FMD_OMD)
428 		packet_log("OMD ");
429 	else if (ret_md == SPU2_RET_FMD_OMD_IV)
430 		packet_log("OMD IV ");
431 	if (ctrl1 & SPU2_RETURN_FD)
432 		packet_log("FD ");
433 	if (ctrl1 & SPU2_RETURN_AAD1)
434 		packet_log("AAD1 ");
435 	if (ctrl1 & SPU2_RETURN_NAAD)
436 		packet_log("NAAD ");
437 	if (ctrl1 & SPU2_RETURN_AAD2)
438 		packet_log("AAD2 ");
439 	if (ctrl1 & SPU2_RETURN_PAY)
440 		packet_log("Payload");
441 	packet_log("\n");
442 }
443 
444 /* Dump FMD ctrl2. The ctrl2 input is in host byte order */
spu2_dump_fmd_ctrl2(u64 ctrl2)445 static void spu2_dump_fmd_ctrl2(u64 ctrl2)
446 {
447 	packet_log(" FMD CTRL2 %#16llx\n", ctrl2);
448 
449 	packet_log("  AAD1 offset %llu length %llu bytes\n",
450 		   ctrl2 & SPU2_AAD1_OFFSET,
451 		   (ctrl2 & SPU2_AAD1_LEN) >> SPU2_AAD1_LEN_SHIFT);
452 	packet_log("  AAD2 offset %llu\n",
453 		   (ctrl2 & SPU2_AAD2_OFFSET) >> SPU2_AAD2_OFFSET_SHIFT);
454 	packet_log("  Payload offset %llu\n",
455 		   (ctrl2 & SPU2_PL_OFFSET) >> SPU2_PL_OFFSET_SHIFT);
456 }
457 
458 /* Dump FMD ctrl3. The ctrl3 input is in host byte order */
spu2_dump_fmd_ctrl3(u64 ctrl3)459 static void spu2_dump_fmd_ctrl3(u64 ctrl3)
460 {
461 	packet_log(" FMD CTRL3 %#16llx\n", ctrl3);
462 
463 	packet_log("  Payload length %llu bytes\n", ctrl3 & SPU2_PL_LEN);
464 	packet_log("  TLS length %llu bytes\n",
465 		   (ctrl3 & SPU2_TLS_LEN) >> SPU2_TLS_LEN_SHIFT);
466 }
467 
spu2_dump_fmd(struct SPU2_FMD * fmd)468 static void spu2_dump_fmd(struct SPU2_FMD *fmd)
469 {
470 	spu2_dump_fmd_ctrl0(le64_to_cpu(fmd->ctrl0));
471 	spu2_dump_fmd_ctrl1(le64_to_cpu(fmd->ctrl1));
472 	spu2_dump_fmd_ctrl2(le64_to_cpu(fmd->ctrl2));
473 	spu2_dump_fmd_ctrl3(le64_to_cpu(fmd->ctrl3));
474 }
475 
spu2_dump_omd(u8 * omd,u16 hash_key_len,u16 ciph_key_len,u16 hash_iv_len,u16 ciph_iv_len)476 static void spu2_dump_omd(u8 *omd, u16 hash_key_len, u16 ciph_key_len,
477 			  u16 hash_iv_len, u16 ciph_iv_len)
478 {
479 	u8 *ptr = omd;
480 
481 	packet_log(" OMD:\n");
482 
483 	if (hash_key_len) {
484 		packet_log("  Hash Key Length %u bytes\n", hash_key_len);
485 		packet_dump("  KEY: ", ptr, hash_key_len);
486 		ptr += hash_key_len;
487 	}
488 
489 	if (ciph_key_len) {
490 		packet_log("  Cipher Key Length %u bytes\n", ciph_key_len);
491 		packet_dump("  KEY: ", ptr, ciph_key_len);
492 		ptr += ciph_key_len;
493 	}
494 
495 	if (hash_iv_len) {
496 		packet_log("  Hash IV Length %u bytes\n", hash_iv_len);
497 		packet_dump("  hash IV: ", ptr, hash_iv_len);
498 		ptr += hash_iv_len;
499 	}
500 
501 	if (ciph_iv_len) {
502 		packet_log("  Cipher IV Length %u bytes\n", ciph_iv_len);
503 		packet_dump("  cipher IV: ", ptr, ciph_iv_len);
504 	}
505 }
506 
507 /* Dump a SPU2 header for debug */
spu2_dump_msg_hdr(u8 * buf,unsigned int buf_len)508 void spu2_dump_msg_hdr(u8 *buf, unsigned int buf_len)
509 {
510 	struct SPU2_FMD *fmd = (struct SPU2_FMD *)buf;
511 	u8 *omd;
512 	u64 ctrl1;
513 	u16 hash_key_len;
514 	u16 ciph_key_len;
515 	u16 hash_iv_len;
516 	u16 ciph_iv_len;
517 	u16 omd_len;
518 
519 	packet_log("\n");
520 	packet_log("SPU2 message header %p len: %u\n", buf, buf_len);
521 
522 	spu2_dump_fmd(fmd);
523 	omd = (u8 *)(fmd + 1);
524 
525 	ctrl1 = le64_to_cpu(fmd->ctrl1);
526 	hash_key_len = (ctrl1 & SPU2_HASH_KEY_LEN) >> SPU2_HASH_KEY_LEN_SHIFT;
527 	ciph_key_len = (ctrl1 & SPU2_CIPH_KEY_LEN) >> SPU2_CIPH_KEY_LEN_SHIFT;
528 	hash_iv_len = 0;
529 	ciph_iv_len = (ctrl1 & SPU2_IV_LEN) >> SPU2_IV_LEN_SHIFT;
530 	spu2_dump_omd(omd, hash_key_len, ciph_key_len, hash_iv_len,
531 		      ciph_iv_len);
532 
533 	/* Double check sanity */
534 	omd_len = hash_key_len + ciph_key_len + hash_iv_len + ciph_iv_len;
535 	if (FMD_SIZE + omd_len != buf_len) {
536 		packet_log
537 		    (" Packet parsed incorrectly. buf_len %u, sum of MD %zu\n",
538 		     buf_len, FMD_SIZE + omd_len);
539 	}
540 	packet_log("\n");
541 }
542 
543 /**
544  * spu2_fmd_init() - At setkey time, initialize the fixed meta data for
545  * subsequent skcipher requests for this context.
546  * @fmd:               Start of FMD field to be written
547  * @spu2_type:         Cipher algorithm
548  * @spu2_mode:         Cipher mode
549  * @cipher_key_len:    Length of cipher key, in bytes
550  * @cipher_iv_len:     Length of cipher initialization vector, in bytes
551  *
552  * Return:  0 (success)
553  */
spu2_fmd_init(struct SPU2_FMD * fmd,enum spu2_cipher_type spu2_type,enum spu2_cipher_mode spu2_mode,u32 cipher_key_len,u32 cipher_iv_len)554 static int spu2_fmd_init(struct SPU2_FMD *fmd,
555 			 enum spu2_cipher_type spu2_type,
556 			 enum spu2_cipher_mode spu2_mode,
557 			 u32 cipher_key_len, u32 cipher_iv_len)
558 {
559 	u64 ctrl0;
560 	u64 ctrl1;
561 	u64 ctrl2;
562 	u64 ctrl3;
563 	u32 aad1_offset;
564 	u32 aad2_offset;
565 	u16 aad1_len = 0;
566 	u64 payload_offset;
567 
568 	ctrl0 = (spu2_type << SPU2_CIPH_TYPE_SHIFT) |
569 	    (spu2_mode << SPU2_CIPH_MODE_SHIFT);
570 
571 	ctrl1 = (cipher_key_len << SPU2_CIPH_KEY_LEN_SHIFT) |
572 	    ((u64)cipher_iv_len << SPU2_IV_LEN_SHIFT) |
573 	    ((u64)SPU2_RET_FMD_ONLY << SPU2_RETURN_MD_SHIFT) | SPU2_RETURN_PAY;
574 
575 	/*
576 	 * AAD1 offset is from start of FD. FD length is always 0 for this
577 	 * driver. So AAD1_offset is always 0.
578 	 */
579 	aad1_offset = 0;
580 	aad2_offset = aad1_offset;
581 	payload_offset = 0;
582 	ctrl2 = aad1_offset |
583 	    (aad1_len << SPU2_AAD1_LEN_SHIFT) |
584 	    (aad2_offset << SPU2_AAD2_OFFSET_SHIFT) |
585 	    (payload_offset << SPU2_PL_OFFSET_SHIFT);
586 
587 	ctrl3 = 0;
588 
589 	fmd->ctrl0 = cpu_to_le64(ctrl0);
590 	fmd->ctrl1 = cpu_to_le64(ctrl1);
591 	fmd->ctrl2 = cpu_to_le64(ctrl2);
592 	fmd->ctrl3 = cpu_to_le64(ctrl3);
593 
594 	return 0;
595 }
596 
597 /**
598  * spu2_fmd_ctrl0_write() - Write ctrl0 field in fixed metadata (FMD) field of
599  * SPU request packet.
600  * @fmd:            Start of FMD field to be written
601  * @is_inbound:     true if decrypting. false if encrypting.
602  * @auth_first:     true if alg authenticates before encrypting
603  * @protocol:       protocol selector
604  * @cipher_type:    cipher algorithm
605  * @cipher_mode:    cipher mode
606  * @auth_type:      authentication type
607  * @auth_mode:      authentication mode
608  */
spu2_fmd_ctrl0_write(struct SPU2_FMD * fmd,bool is_inbound,bool auth_first,enum spu2_proto_sel protocol,enum spu2_cipher_type cipher_type,enum spu2_cipher_mode cipher_mode,enum spu2_hash_type auth_type,enum spu2_hash_mode auth_mode)609 static void spu2_fmd_ctrl0_write(struct SPU2_FMD *fmd,
610 				 bool is_inbound, bool auth_first,
611 				 enum spu2_proto_sel protocol,
612 				 enum spu2_cipher_type cipher_type,
613 				 enum spu2_cipher_mode cipher_mode,
614 				 enum spu2_hash_type auth_type,
615 				 enum spu2_hash_mode auth_mode)
616 {
617 	u64 ctrl0 = 0;
618 
619 	if ((cipher_type != SPU2_CIPHER_TYPE_NONE) && !is_inbound)
620 		ctrl0 |= SPU2_CIPH_ENCRYPT_EN;
621 
622 	ctrl0 |= ((u64)cipher_type << SPU2_CIPH_TYPE_SHIFT) |
623 	    ((u64)cipher_mode << SPU2_CIPH_MODE_SHIFT);
624 
625 	if (protocol)
626 		ctrl0 |= (u64)protocol << SPU2_PROTO_SEL_SHIFT;
627 
628 	if (auth_first)
629 		ctrl0 |= SPU2_HASH_FIRST;
630 
631 	if (is_inbound && (auth_type != SPU2_HASH_TYPE_NONE))
632 		ctrl0 |= SPU2_CHK_TAG;
633 
634 	ctrl0 |= (((u64)auth_type << SPU2_HASH_TYPE_SHIFT) |
635 		  ((u64)auth_mode << SPU2_HASH_MODE_SHIFT));
636 
637 	fmd->ctrl0 = cpu_to_le64(ctrl0);
638 }
639 
640 /**
641  * spu2_fmd_ctrl1_write() - Write ctrl1 field in fixed metadata (FMD) field of
642  * SPU request packet.
643  * @fmd:            Start of FMD field to be written
644  * @is_inbound:     true if decrypting. false if encrypting.
645  * @assoc_size:     Length of additional associated data, in bytes
646  * @auth_key_len:   Length of authentication key, in bytes
647  * @cipher_key_len: Length of cipher key, in bytes
648  * @gen_iv:         If true, hw generates IV and returns in response
649  * @hash_iv:        IV participates in hash. Used for IPSEC and TLS.
650  * @return_iv:      Return IV in output packet before payload
651  * @ret_iv_len:     Length of IV returned from SPU, in bytes
652  * @ret_iv_offset:  Offset into full IV of start of returned IV
653  * @cipher_iv_len:  Length of input cipher IV, in bytes
654  * @digest_size:    Length of digest (aka, hash tag or ICV), in bytes
655  * @return_payload: Return payload in SPU response
656  * @return_md : return metadata in SPU response
657  *
658  * Packet can have AAD2 w/o AAD1. For algorithms currently supported,
659  * associated data goes in AAD2.
660  */
spu2_fmd_ctrl1_write(struct SPU2_FMD * fmd,bool is_inbound,u64 assoc_size,u64 auth_key_len,u64 cipher_key_len,bool gen_iv,bool hash_iv,bool return_iv,u64 ret_iv_len,u64 ret_iv_offset,u64 cipher_iv_len,u64 digest_size,bool return_payload,bool return_md)661 static void spu2_fmd_ctrl1_write(struct SPU2_FMD *fmd, bool is_inbound,
662 				 u64 assoc_size,
663 				 u64 auth_key_len, u64 cipher_key_len,
664 				 bool gen_iv, bool hash_iv, bool return_iv,
665 				 u64 ret_iv_len, u64 ret_iv_offset,
666 				 u64 cipher_iv_len, u64 digest_size,
667 				 bool return_payload, bool return_md)
668 {
669 	u64 ctrl1 = 0;
670 
671 	if (is_inbound && digest_size)
672 		ctrl1 |= SPU2_TAG_LOC;
673 
674 	if (assoc_size) {
675 		ctrl1 |= SPU2_HAS_AAD2;
676 		ctrl1 |= SPU2_RETURN_AAD2;  /* need aad2 for gcm aes esp */
677 	}
678 
679 	if (auth_key_len)
680 		ctrl1 |= ((auth_key_len << SPU2_HASH_KEY_LEN_SHIFT) &
681 			  SPU2_HASH_KEY_LEN);
682 
683 	if (cipher_key_len)
684 		ctrl1 |= ((cipher_key_len << SPU2_CIPH_KEY_LEN_SHIFT) &
685 			  SPU2_CIPH_KEY_LEN);
686 
687 	if (gen_iv)
688 		ctrl1 |= SPU2_GENIV;
689 
690 	if (hash_iv)
691 		ctrl1 |= SPU2_HASH_IV;
692 
693 	if (return_iv) {
694 		ctrl1 |= SPU2_RET_IV;
695 		ctrl1 |= ret_iv_len << SPU2_RET_IV_LEN_SHIFT;
696 		ctrl1 |= ret_iv_offset << SPU2_IV_OFFSET_SHIFT;
697 	}
698 
699 	ctrl1 |= ((cipher_iv_len << SPU2_IV_LEN_SHIFT) & SPU2_IV_LEN);
700 
701 	if (digest_size)
702 		ctrl1 |= ((digest_size << SPU2_HASH_TAG_LEN_SHIFT) &
703 			  SPU2_HASH_TAG_LEN);
704 
705 	/* Let's ask for the output pkt to include FMD, but don't need to
706 	 * get keys and IVs back in OMD.
707 	 */
708 	if (return_md)
709 		ctrl1 |= ((u64)SPU2_RET_FMD_ONLY << SPU2_RETURN_MD_SHIFT);
710 	else
711 		ctrl1 |= ((u64)SPU2_RET_NO_MD << SPU2_RETURN_MD_SHIFT);
712 
713 	/* Crypto API does not get assoc data back. So no need for AAD2. */
714 
715 	if (return_payload)
716 		ctrl1 |= SPU2_RETURN_PAY;
717 
718 	fmd->ctrl1 = cpu_to_le64(ctrl1);
719 }
720 
721 /**
722  * spu2_fmd_ctrl2_write() - Set the ctrl2 field in the fixed metadata field of
723  * SPU2 header.
724  * @fmd:            Start of FMD field to be written
725  * @cipher_offset:  Number of bytes from Start of Packet (end of FD field) where
726  *                  data to be encrypted or decrypted begins
727  * @auth_key_len:   Length of authentication key, in bytes
728  * @auth_iv_len:    Length of authentication initialization vector, in bytes
729  * @cipher_key_len: Length of cipher key, in bytes
730  * @cipher_iv_len:  Length of cipher IV, in bytes
731  */
spu2_fmd_ctrl2_write(struct SPU2_FMD * fmd,u64 cipher_offset,u64 auth_key_len,u64 auth_iv_len,u64 cipher_key_len,u64 cipher_iv_len)732 static void spu2_fmd_ctrl2_write(struct SPU2_FMD *fmd, u64 cipher_offset,
733 				 u64 auth_key_len, u64 auth_iv_len,
734 				 u64 cipher_key_len, u64 cipher_iv_len)
735 {
736 	u64 ctrl2;
737 	u64 aad1_offset;
738 	u64 aad2_offset;
739 	u16 aad1_len = 0;
740 	u64 payload_offset;
741 
742 	/* AAD1 offset is from start of FD. FD length always 0. */
743 	aad1_offset = 0;
744 
745 	aad2_offset = aad1_offset;
746 	payload_offset = cipher_offset;
747 	ctrl2 = aad1_offset |
748 	    (aad1_len << SPU2_AAD1_LEN_SHIFT) |
749 	    (aad2_offset << SPU2_AAD2_OFFSET_SHIFT) |
750 	    (payload_offset << SPU2_PL_OFFSET_SHIFT);
751 
752 	fmd->ctrl2 = cpu_to_le64(ctrl2);
753 }
754 
755 /**
756  * spu2_fmd_ctrl3_write() - Set the ctrl3 field in FMD
757  * @fmd:          Fixed meta data. First field in SPU2 msg header.
758  * @payload_len:  Length of payload, in bytes
759  */
spu2_fmd_ctrl3_write(struct SPU2_FMD * fmd,u64 payload_len)760 static void spu2_fmd_ctrl3_write(struct SPU2_FMD *fmd, u64 payload_len)
761 {
762 	u64 ctrl3;
763 
764 	ctrl3 = payload_len & SPU2_PL_LEN;
765 
766 	fmd->ctrl3 = cpu_to_le64(ctrl3);
767 }
768 
769 /**
770  * spu2_ctx_max_payload() - Determine the maximum length of the payload for a
771  * SPU message for a given cipher and hash alg context.
772  * @cipher_alg:		The cipher algorithm
773  * @cipher_mode:	The cipher mode
774  * @blocksize:		The size of a block of data for this algo
775  *
776  * For SPU2, the hardware generally ignores the PayloadLen field in ctrl3 of
777  * FMD and just keeps computing until it receives a DMA descriptor with the EOF
778  * flag set. So we consider the max payload to be infinite. AES CCM is an
779  * exception.
780  *
781  * Return: Max payload length in bytes
782  */
spu2_ctx_max_payload(enum spu_cipher_alg cipher_alg,enum spu_cipher_mode cipher_mode,unsigned int blocksize)783 u32 spu2_ctx_max_payload(enum spu_cipher_alg cipher_alg,
784 			 enum spu_cipher_mode cipher_mode,
785 			 unsigned int blocksize)
786 {
787 	if ((cipher_alg == CIPHER_ALG_AES) &&
788 	    (cipher_mode == CIPHER_MODE_CCM)) {
789 		u32 excess = SPU2_MAX_PAYLOAD % blocksize;
790 
791 		return SPU2_MAX_PAYLOAD - excess;
792 	} else {
793 		return SPU_MAX_PAYLOAD_INF;
794 	}
795 }
796 
797 /**
798  * spu2_payload_length() -  Given a SPU2 message header, extract the payload
799  * length.
800  * @spu_hdr:  Start of SPU message header (FMD)
801  *
802  * Return: payload length, in bytes
803  */
spu2_payload_length(u8 * spu_hdr)804 u32 spu2_payload_length(u8 *spu_hdr)
805 {
806 	struct SPU2_FMD *fmd = (struct SPU2_FMD *)spu_hdr;
807 	u32 pl_len;
808 	u64 ctrl3;
809 
810 	ctrl3 = le64_to_cpu(fmd->ctrl3);
811 	pl_len = ctrl3 & SPU2_PL_LEN;
812 
813 	return pl_len;
814 }
815 
816 /**
817  * spu2_response_hdr_len() - Determine the expected length of a SPU response
818  * header.
819  * @auth_key_len:  Length of authentication key, in bytes
820  * @enc_key_len:   Length of encryption key, in bytes
821  * @is_hash:       Unused
822  *
823  * For SPU2, includes just FMD. OMD is never requested.
824  *
825  * Return: Length of FMD, in bytes
826  */
spu2_response_hdr_len(u16 auth_key_len,u16 enc_key_len,bool is_hash)827 u16 spu2_response_hdr_len(u16 auth_key_len, u16 enc_key_len, bool is_hash)
828 {
829 	return FMD_SIZE;
830 }
831 
832 /**
833  * spu2_hash_pad_len() - Calculate the length of hash padding required to extend
834  * data to a full block size.
835  * @hash_alg:        hash algorithm
836  * @hash_mode:       hash mode
837  * @chunksize:       length of data, in bytes
838  * @hash_block_size: size of a hash block, in bytes
839  *
840  * SPU2 hardware does all hash padding
841  *
842  * Return:  length of hash pad in bytes
843  */
spu2_hash_pad_len(enum hash_alg hash_alg,enum hash_mode hash_mode,u32 chunksize,u16 hash_block_size)844 u16 spu2_hash_pad_len(enum hash_alg hash_alg, enum hash_mode hash_mode,
845 		      u32 chunksize, u16 hash_block_size)
846 {
847 	return 0;
848 }
849 
850 /**
851  * spu2_gcm_ccm_pad_len() -  Determine the length of GCM/CCM padding for either
852  * the AAD field or the data.
853  * @cipher_mode:  Unused
854  * @data_size:    Unused
855  *
856  * Return:  0. Unlike SPU-M, SPU2 hardware does any GCM/CCM padding required.
857  */
spu2_gcm_ccm_pad_len(enum spu_cipher_mode cipher_mode,unsigned int data_size)858 u32 spu2_gcm_ccm_pad_len(enum spu_cipher_mode cipher_mode,
859 			 unsigned int data_size)
860 {
861 	return 0;
862 }
863 
864 /**
865  * spu2_assoc_resp_len() - Determine the size of the AAD2 buffer needed to catch
866  * associated data in a SPU2 output packet.
867  * @cipher_mode:   cipher mode
868  * @assoc_len:     length of additional associated data, in bytes
869  * @iv_len:        length of initialization vector, in bytes
870  * @is_encrypt:    true if encrypting. false if decrypt.
871  *
872  * Return: Length of buffer to catch associated data in response
873  */
spu2_assoc_resp_len(enum spu_cipher_mode cipher_mode,unsigned int assoc_len,unsigned int iv_len,bool is_encrypt)874 u32 spu2_assoc_resp_len(enum spu_cipher_mode cipher_mode,
875 			unsigned int assoc_len, unsigned int iv_len,
876 			bool is_encrypt)
877 {
878 	u32 resp_len = assoc_len;
879 
880 	if (is_encrypt)
881 		/* gcm aes esp has to write 8-byte IV in response */
882 		resp_len += iv_len;
883 	return resp_len;
884 }
885 
886 /**
887  * spu2_aead_ivlen() - Calculate the length of the AEAD IV to be included
888  * in a SPU request after the AAD and before the payload.
889  * @cipher_mode:  cipher mode
890  * @iv_len:   initialization vector length in bytes
891  *
892  * For SPU2, AEAD IV is included in OMD and does not need to be repeated
893  * prior to the payload.
894  *
895  * Return: Length of AEAD IV in bytes
896  */
spu2_aead_ivlen(enum spu_cipher_mode cipher_mode,u16 iv_len)897 u8 spu2_aead_ivlen(enum spu_cipher_mode cipher_mode, u16 iv_len)
898 {
899 	return 0;
900 }
901 
902 /**
903  * spu2_hash_type() - Determine the type of hash operation.
904  * @src_sent:  The number of bytes in the current request that have already
905  *             been sent to the SPU to be hashed.
906  *
907  * SPU2 always does a FULL hash operation
908  */
spu2_hash_type(u32 src_sent)909 enum hash_type spu2_hash_type(u32 src_sent)
910 {
911 	return HASH_TYPE_FULL;
912 }
913 
914 /**
915  * spu2_digest_size() - Determine the size of a hash digest to expect the SPU to
916  * return.
917  * @alg_digest_size: Number of bytes in the final digest for the given algo
918  * @alg:             The hash algorithm
919  * @htype:           Type of hash operation (init, update, full, etc)
920  *
921  */
spu2_digest_size(u32 alg_digest_size,enum hash_alg alg,enum hash_type htype)922 u32 spu2_digest_size(u32 alg_digest_size, enum hash_alg alg,
923 		     enum hash_type htype)
924 {
925 	return alg_digest_size;
926 }
927 
928 /**
929  * spu2_create_request() - Build a SPU2 request message header, includint FMD and
930  * OMD.
931  * @spu_hdr: Start of buffer where SPU request header is to be written
932  * @req_opts: SPU request message options
933  * @cipher_parms: Parameters related to cipher algorithm
934  * @hash_parms:   Parameters related to hash algorithm
935  * @aead_parms:   Parameters related to AEAD operation
936  * @data_size:    Length of data to be encrypted or authenticated. If AEAD, does
937  *		  not include length of AAD.
938  *
939  * Construct the message starting at spu_hdr. Caller should allocate this buffer
940  * in DMA-able memory at least SPU_HEADER_ALLOC_LEN bytes long.
941  *
942  * Return: the length of the SPU header in bytes. 0 if an error occurs.
943  */
spu2_create_request(u8 * spu_hdr,struct spu_request_opts * req_opts,struct spu_cipher_parms * cipher_parms,struct spu_hash_parms * hash_parms,struct spu_aead_parms * aead_parms,unsigned int data_size)944 u32 spu2_create_request(u8 *spu_hdr,
945 			struct spu_request_opts *req_opts,
946 			struct spu_cipher_parms *cipher_parms,
947 			struct spu_hash_parms *hash_parms,
948 			struct spu_aead_parms *aead_parms,
949 			unsigned int data_size)
950 {
951 	struct SPU2_FMD *fmd;
952 	u8 *ptr;
953 	unsigned int buf_len;
954 	int err;
955 	enum spu2_cipher_type spu2_ciph_type = SPU2_CIPHER_TYPE_NONE;
956 	enum spu2_cipher_mode spu2_ciph_mode;
957 	enum spu2_hash_type spu2_auth_type = SPU2_HASH_TYPE_NONE;
958 	enum spu2_hash_mode spu2_auth_mode;
959 	bool return_md = true;
960 	enum spu2_proto_sel proto = SPU2_PROTO_RESV;
961 
962 	/* size of the payload */
963 	unsigned int payload_len =
964 	    hash_parms->prebuf_len + data_size + hash_parms->pad_len -
965 	    ((req_opts->is_aead && req_opts->is_inbound) ?
966 	     hash_parms->digestsize : 0);
967 
968 	/* offset of prebuf or data from start of AAD2 */
969 	unsigned int cipher_offset = aead_parms->assoc_size +
970 			aead_parms->aad_pad_len + aead_parms->iv_len;
971 
972 	/* total size of the data following OMD (without STAT word padding) */
973 	unsigned int real_db_size = spu_real_db_size(aead_parms->assoc_size,
974 						 aead_parms->iv_len,
975 						 hash_parms->prebuf_len,
976 						 data_size,
977 						 aead_parms->aad_pad_len,
978 						 aead_parms->data_pad_len,
979 						 hash_parms->pad_len);
980 	unsigned int assoc_size = aead_parms->assoc_size;
981 
982 	if (req_opts->is_aead &&
983 	    (cipher_parms->alg == CIPHER_ALG_AES) &&
984 	    (cipher_parms->mode == CIPHER_MODE_GCM))
985 		/*
986 		 * On SPU 2, aes gcm cipher first on encrypt, auth first on
987 		 * decrypt
988 		 */
989 		req_opts->auth_first = req_opts->is_inbound;
990 
991 	/* and do opposite for ccm (auth 1st on encrypt) */
992 	if (req_opts->is_aead &&
993 	    (cipher_parms->alg == CIPHER_ALG_AES) &&
994 	    (cipher_parms->mode == CIPHER_MODE_CCM))
995 		req_opts->auth_first = !req_opts->is_inbound;
996 
997 	flow_log("%s()\n", __func__);
998 	flow_log("  in:%u authFirst:%u\n",
999 		 req_opts->is_inbound, req_opts->auth_first);
1000 	flow_log("  cipher alg:%u mode:%u type %u\n", cipher_parms->alg,
1001 		 cipher_parms->mode, cipher_parms->type);
1002 	flow_log("  is_esp: %s\n", req_opts->is_esp ? "yes" : "no");
1003 	flow_log("    key: %d\n", cipher_parms->key_len);
1004 	flow_dump("    key: ", cipher_parms->key_buf, cipher_parms->key_len);
1005 	flow_log("    iv: %d\n", cipher_parms->iv_len);
1006 	flow_dump("    iv: ", cipher_parms->iv_buf, cipher_parms->iv_len);
1007 	flow_log("  auth alg:%u mode:%u type %u\n",
1008 		 hash_parms->alg, hash_parms->mode, hash_parms->type);
1009 	flow_log("  digestsize: %u\n", hash_parms->digestsize);
1010 	flow_log("  authkey: %d\n", hash_parms->key_len);
1011 	flow_dump("  authkey: ", hash_parms->key_buf, hash_parms->key_len);
1012 	flow_log("  assoc_size:%u\n", assoc_size);
1013 	flow_log("  prebuf_len:%u\n", hash_parms->prebuf_len);
1014 	flow_log("  data_size:%u\n", data_size);
1015 	flow_log("  hash_pad_len:%u\n", hash_parms->pad_len);
1016 	flow_log("  real_db_size:%u\n", real_db_size);
1017 	flow_log("  cipher_offset:%u payload_len:%u\n",
1018 		 cipher_offset, payload_len);
1019 	flow_log("  aead_iv: %u\n", aead_parms->iv_len);
1020 
1021 	/* Convert to spu2 values for cipher alg, hash alg */
1022 	err = spu2_cipher_xlate(cipher_parms->alg, cipher_parms->mode,
1023 				cipher_parms->type,
1024 				&spu2_ciph_type, &spu2_ciph_mode);
1025 
1026 	/* If we are doing GCM hashing only - either via rfc4543 transform
1027 	 * or because we happen to do GCM with AAD only and no payload - we
1028 	 * need to configure hardware to use hash key rather than cipher key
1029 	 * and put data into payload.  This is because unlike SPU-M, running
1030 	 * GCM cipher with 0 size payload is not permitted.
1031 	 */
1032 	if ((req_opts->is_rfc4543) ||
1033 	    ((spu2_ciph_mode == SPU2_CIPHER_MODE_GCM) &&
1034 	    (payload_len == 0))) {
1035 		/* Use hashing (only) and set up hash key */
1036 		spu2_ciph_type = SPU2_CIPHER_TYPE_NONE;
1037 		hash_parms->key_len = cipher_parms->key_len;
1038 		memcpy(hash_parms->key_buf, cipher_parms->key_buf,
1039 		       cipher_parms->key_len);
1040 		cipher_parms->key_len = 0;
1041 
1042 		if (req_opts->is_rfc4543)
1043 			payload_len += assoc_size;
1044 		else
1045 			payload_len = assoc_size;
1046 		cipher_offset = 0;
1047 		assoc_size = 0;
1048 	}
1049 
1050 	if (err)
1051 		return 0;
1052 
1053 	flow_log("spu2 cipher type %s, cipher mode %s\n",
1054 		 spu2_ciph_type_name(spu2_ciph_type),
1055 		 spu2_ciph_mode_name(spu2_ciph_mode));
1056 
1057 	err = spu2_hash_xlate(hash_parms->alg, hash_parms->mode,
1058 			      hash_parms->type,
1059 			      cipher_parms->type,
1060 			      &spu2_auth_type, &spu2_auth_mode);
1061 	if (err)
1062 		return 0;
1063 
1064 	flow_log("spu2 hash type %s, hash mode %s\n",
1065 		 spu2_hash_type_name(spu2_auth_type),
1066 		 spu2_hash_mode_name(spu2_auth_mode));
1067 
1068 	fmd = (struct SPU2_FMD *)spu_hdr;
1069 
1070 	spu2_fmd_ctrl0_write(fmd, req_opts->is_inbound, req_opts->auth_first,
1071 			     proto, spu2_ciph_type, spu2_ciph_mode,
1072 			     spu2_auth_type, spu2_auth_mode);
1073 
1074 	spu2_fmd_ctrl1_write(fmd, req_opts->is_inbound, assoc_size,
1075 			     hash_parms->key_len, cipher_parms->key_len,
1076 			     false, false,
1077 			     aead_parms->return_iv, aead_parms->ret_iv_len,
1078 			     aead_parms->ret_iv_off,
1079 			     cipher_parms->iv_len, hash_parms->digestsize,
1080 			     !req_opts->bd_suppress, return_md);
1081 
1082 	spu2_fmd_ctrl2_write(fmd, cipher_offset, hash_parms->key_len, 0,
1083 			     cipher_parms->key_len, cipher_parms->iv_len);
1084 
1085 	spu2_fmd_ctrl3_write(fmd, payload_len);
1086 
1087 	ptr = (u8 *)(fmd + 1);
1088 	buf_len = sizeof(struct SPU2_FMD);
1089 
1090 	/* Write OMD */
1091 	if (hash_parms->key_len) {
1092 		memcpy(ptr, hash_parms->key_buf, hash_parms->key_len);
1093 		ptr += hash_parms->key_len;
1094 		buf_len += hash_parms->key_len;
1095 	}
1096 	if (cipher_parms->key_len) {
1097 		memcpy(ptr, cipher_parms->key_buf, cipher_parms->key_len);
1098 		ptr += cipher_parms->key_len;
1099 		buf_len += cipher_parms->key_len;
1100 	}
1101 	if (cipher_parms->iv_len) {
1102 		memcpy(ptr, cipher_parms->iv_buf, cipher_parms->iv_len);
1103 		ptr += cipher_parms->iv_len;
1104 		buf_len += cipher_parms->iv_len;
1105 	}
1106 
1107 	packet_dump("  SPU request header: ", spu_hdr, buf_len);
1108 
1109 	return buf_len;
1110 }
1111 
1112 /**
1113  * spu2_cipher_req_init() - Build an skcipher SPU2 request message header,
1114  * including FMD and OMD.
1115  * @spu_hdr:       Location of start of SPU request (FMD field)
1116  * @cipher_parms:  Parameters describing cipher request
1117  *
1118  * Called at setkey time to initialize a msg header that can be reused for all
1119  * subsequent skcipher requests. Construct the message starting at spu_hdr.
1120  * Caller should allocate this buffer in DMA-able memory at least
1121  * SPU_HEADER_ALLOC_LEN bytes long.
1122  *
1123  * Return: the total length of the SPU header (FMD and OMD) in bytes. 0 if an
1124  * error occurs.
1125  */
spu2_cipher_req_init(u8 * spu_hdr,struct spu_cipher_parms * cipher_parms)1126 u16 spu2_cipher_req_init(u8 *spu_hdr, struct spu_cipher_parms *cipher_parms)
1127 {
1128 	struct SPU2_FMD *fmd;
1129 	u8 *omd;
1130 	enum spu2_cipher_type spu2_type = SPU2_CIPHER_TYPE_NONE;
1131 	enum spu2_cipher_mode spu2_mode;
1132 	int err;
1133 
1134 	flow_log("%s()\n", __func__);
1135 	flow_log("  cipher alg:%u mode:%u type %u\n", cipher_parms->alg,
1136 		 cipher_parms->mode, cipher_parms->type);
1137 	flow_log("  cipher_iv_len: %u\n", cipher_parms->iv_len);
1138 	flow_log("    key: %d\n", cipher_parms->key_len);
1139 	flow_dump("    key: ", cipher_parms->key_buf, cipher_parms->key_len);
1140 
1141 	/* Convert to spu2 values */
1142 	err = spu2_cipher_xlate(cipher_parms->alg, cipher_parms->mode,
1143 				cipher_parms->type, &spu2_type, &spu2_mode);
1144 	if (err)
1145 		return 0;
1146 
1147 	flow_log("spu2 cipher type %s, cipher mode %s\n",
1148 		 spu2_ciph_type_name(spu2_type),
1149 		 spu2_ciph_mode_name(spu2_mode));
1150 
1151 	/* Construct the FMD header */
1152 	fmd = (struct SPU2_FMD *)spu_hdr;
1153 	err = spu2_fmd_init(fmd, spu2_type, spu2_mode, cipher_parms->key_len,
1154 			    cipher_parms->iv_len);
1155 	if (err)
1156 		return 0;
1157 
1158 	/* Write cipher key to OMD */
1159 	omd = (u8 *)(fmd + 1);
1160 	if (cipher_parms->key_buf && cipher_parms->key_len)
1161 		memcpy(omd, cipher_parms->key_buf, cipher_parms->key_len);
1162 
1163 	packet_dump("  SPU request header: ", spu_hdr,
1164 		    FMD_SIZE + cipher_parms->key_len + cipher_parms->iv_len);
1165 
1166 	return FMD_SIZE + cipher_parms->key_len + cipher_parms->iv_len;
1167 }
1168 
1169 /**
1170  * spu2_cipher_req_finish() - Finish building a SPU request message header for a
1171  * block cipher request.
1172  * @spu_hdr:         Start of the request message header (MH field)
1173  * @spu_req_hdr_len: Length in bytes of the SPU request header
1174  * @is_inbound:      0 encrypt, 1 decrypt
1175  * @cipher_parms:    Parameters describing cipher operation to be performed
1176  * @data_size:       Length of the data in the BD field
1177  *
1178  * Assumes much of the header was already filled in at setkey() time in
1179  * spu_cipher_req_init().
1180  * spu_cipher_req_init() fills in the encryption key.
1181  */
spu2_cipher_req_finish(u8 * spu_hdr,u16 spu_req_hdr_len,unsigned int is_inbound,struct spu_cipher_parms * cipher_parms,unsigned int data_size)1182 void spu2_cipher_req_finish(u8 *spu_hdr,
1183 			    u16 spu_req_hdr_len,
1184 			    unsigned int is_inbound,
1185 			    struct spu_cipher_parms *cipher_parms,
1186 			    unsigned int data_size)
1187 {
1188 	struct SPU2_FMD *fmd;
1189 	u8 *omd;		/* start of optional metadata */
1190 	u64 ctrl0;
1191 	u64 ctrl3;
1192 
1193 	flow_log("%s()\n", __func__);
1194 	flow_log(" in: %u\n", is_inbound);
1195 	flow_log(" cipher alg: %u, cipher_type: %u\n", cipher_parms->alg,
1196 		 cipher_parms->type);
1197 	flow_log(" iv len: %d\n", cipher_parms->iv_len);
1198 	flow_dump("    iv: ", cipher_parms->iv_buf, cipher_parms->iv_len);
1199 	flow_log(" data_size: %u\n", data_size);
1200 
1201 	fmd = (struct SPU2_FMD *)spu_hdr;
1202 	omd = (u8 *)(fmd + 1);
1203 
1204 	/*
1205 	 * FMD ctrl0 was initialized at setkey time. update it to indicate
1206 	 * whether we are encrypting or decrypting.
1207 	 */
1208 	ctrl0 = le64_to_cpu(fmd->ctrl0);
1209 	if (is_inbound)
1210 		ctrl0 &= ~SPU2_CIPH_ENCRYPT_EN;	/* decrypt */
1211 	else
1212 		ctrl0 |= SPU2_CIPH_ENCRYPT_EN;	/* encrypt */
1213 	fmd->ctrl0 = cpu_to_le64(ctrl0);
1214 
1215 	if (cipher_parms->alg && cipher_parms->iv_buf && cipher_parms->iv_len) {
1216 		/* cipher iv provided so put it in here */
1217 		memcpy(omd + cipher_parms->key_len, cipher_parms->iv_buf,
1218 		       cipher_parms->iv_len);
1219 	}
1220 
1221 	ctrl3 = le64_to_cpu(fmd->ctrl3);
1222 	data_size &= SPU2_PL_LEN;
1223 	ctrl3 |= data_size;
1224 	fmd->ctrl3 = cpu_to_le64(ctrl3);
1225 
1226 	packet_dump("  SPU request header: ", spu_hdr, spu_req_hdr_len);
1227 }
1228 
1229 /**
1230  * spu2_request_pad() - Create pad bytes at the end of the data.
1231  * @pad_start:      Start of buffer where pad bytes are to be written
1232  * @gcm_padding:    Length of GCM padding, in bytes
1233  * @hash_pad_len:   Number of bytes of padding extend data to full block
1234  * @auth_alg:       Authentication algorithm
1235  * @auth_mode:      Authentication mode
1236  * @total_sent:     Length inserted at end of hash pad
1237  * @status_padding: Number of bytes of padding to align STATUS word
1238  *
1239  * There may be three forms of pad:
1240  *  1. GCM pad - for GCM mode ciphers, pad to 16-byte alignment
1241  *  2. hash pad - pad to a block length, with 0x80 data terminator and
1242  *                size at the end
1243  *  3. STAT pad - to ensure the STAT field is 4-byte aligned
1244  */
spu2_request_pad(u8 * pad_start,u32 gcm_padding,u32 hash_pad_len,enum hash_alg auth_alg,enum hash_mode auth_mode,unsigned int total_sent,u32 status_padding)1245 void spu2_request_pad(u8 *pad_start, u32 gcm_padding, u32 hash_pad_len,
1246 		      enum hash_alg auth_alg, enum hash_mode auth_mode,
1247 		      unsigned int total_sent, u32 status_padding)
1248 {
1249 	u8 *ptr = pad_start;
1250 
1251 	/* fix data alignent for GCM */
1252 	if (gcm_padding > 0) {
1253 		flow_log("  GCM: padding to 16 byte alignment: %u bytes\n",
1254 			 gcm_padding);
1255 		memset(ptr, 0, gcm_padding);
1256 		ptr += gcm_padding;
1257 	}
1258 
1259 	if (hash_pad_len > 0) {
1260 		/* clear the padding section */
1261 		memset(ptr, 0, hash_pad_len);
1262 
1263 		/* terminate the data */
1264 		*ptr = 0x80;
1265 		ptr += (hash_pad_len - sizeof(u64));
1266 
1267 		/* add the size at the end as required per alg */
1268 		if (auth_alg == HASH_ALG_MD5)
1269 			*(__le64 *)ptr = cpu_to_le64(total_sent * 8ull);
1270 		else		/* SHA1, SHA2-224, SHA2-256 */
1271 			*(__be64 *)ptr = cpu_to_be64(total_sent * 8ull);
1272 		ptr += sizeof(u64);
1273 	}
1274 
1275 	/* pad to a 4byte alignment for STAT */
1276 	if (status_padding > 0) {
1277 		flow_log("  STAT: padding to 4 byte alignment: %u bytes\n",
1278 			 status_padding);
1279 
1280 		memset(ptr, 0, status_padding);
1281 		ptr += status_padding;
1282 	}
1283 }
1284 
1285 /**
1286  * spu2_xts_tweak_in_payload() - Indicate that SPU2 does NOT place the XTS
1287  * tweak field in the packet payload (it uses IV instead)
1288  *
1289  * Return: 0
1290  */
spu2_xts_tweak_in_payload(void)1291 u8 spu2_xts_tweak_in_payload(void)
1292 {
1293 	return 0;
1294 }
1295 
1296 /**
1297  * spu2_tx_status_len() - Return the length of the STATUS field in a SPU
1298  * response message.
1299  *
1300  * Return: Length of STATUS field in bytes.
1301  */
spu2_tx_status_len(void)1302 u8 spu2_tx_status_len(void)
1303 {
1304 	return SPU2_TX_STATUS_LEN;
1305 }
1306 
1307 /**
1308  * spu2_rx_status_len() - Return the length of the STATUS field in a SPU
1309  * response message.
1310  *
1311  * Return: Length of STATUS field in bytes.
1312  */
spu2_rx_status_len(void)1313 u8 spu2_rx_status_len(void)
1314 {
1315 	return SPU2_RX_STATUS_LEN;
1316 }
1317 
1318 /**
1319  * spu2_status_process() - Process the status from a SPU response message.
1320  * @statp:  start of STATUS word
1321  *
1322  * Return:  0 - if status is good and response should be processed
1323  *         !0 - status indicates an error and response is invalid
1324  */
spu2_status_process(u8 * statp)1325 int spu2_status_process(u8 *statp)
1326 {
1327 	/* SPU2 status is 2 bytes by default - SPU_RX_STATUS_LEN */
1328 	u16 status = le16_to_cpu(*(__le16 *)statp);
1329 
1330 	if (status == 0)
1331 		return 0;
1332 
1333 	flow_log("rx status is %#x\n", status);
1334 	if (status == SPU2_INVALID_ICV)
1335 		return SPU_INVALID_ICV;
1336 
1337 	return -EBADMSG;
1338 }
1339 
1340 /**
1341  * spu2_ccm_update_iv() - Update the IV as per the requirements for CCM mode.
1342  *
1343  * @digestsize:		Digest size of this request
1344  * @cipher_parms:	(pointer to) cipher parmaeters, includes IV buf & IV len
1345  * @assoclen:		Length of AAD data
1346  * @chunksize:		length of input data to be sent in this req
1347  * @is_encrypt:		true if this is an output/encrypt operation
1348  * @is_esp:		true if this is an ESP / RFC4309 operation
1349  *
1350  */
spu2_ccm_update_iv(unsigned int digestsize,struct spu_cipher_parms * cipher_parms,unsigned int assoclen,unsigned int chunksize,bool is_encrypt,bool is_esp)1351 void spu2_ccm_update_iv(unsigned int digestsize,
1352 			struct spu_cipher_parms *cipher_parms,
1353 			unsigned int assoclen, unsigned int chunksize,
1354 			bool is_encrypt, bool is_esp)
1355 {
1356 	int L;  /* size of length field, in bytes */
1357 
1358 	/*
1359 	 * In RFC4309 mode, L is fixed at 4 bytes; otherwise, IV from
1360 	 * testmgr contains (L-1) in bottom 3 bits of first byte,
1361 	 * per RFC 3610.
1362 	 */
1363 	if (is_esp)
1364 		L = CCM_ESP_L_VALUE;
1365 	else
1366 		L = ((cipher_parms->iv_buf[0] & CCM_B0_L_PRIME) >>
1367 		      CCM_B0_L_PRIME_SHIFT) + 1;
1368 
1369 	/* SPU2 doesn't want these length bytes nor the first byte... */
1370 	cipher_parms->iv_len -= (1 + L);
1371 	memmove(cipher_parms->iv_buf, &cipher_parms->iv_buf[1],
1372 		cipher_parms->iv_len);
1373 }
1374 
1375 /**
1376  * spu2_wordalign_padlen() - SPU2 does not require padding.
1377  * @data_size: length of data field in bytes
1378  *
1379  * Return: length of status field padding, in bytes (always 0 on SPU2)
1380  */
spu2_wordalign_padlen(u32 data_size)1381 u32 spu2_wordalign_padlen(u32 data_size)
1382 {
1383 	return 0;
1384 }
1385