xref: /illumos-gate/usr/src/uts/common/crypto/io/dca_rsa.c (revision 24da5b34f49324ed742a340010ed5bd3d4e06625)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 /*
30  * Deimos - cryptographic acceleration based upon Broadcom 582x.
31  */
32 
33 #include <sys/types.h>
34 #include <sys/ddi.h>
35 #include <sys/sunddi.h>
36 #include <sys/kmem.h>
37 #include <sys/note.h>
38 #include <sys/crypto/spi.h>
39 #include <sys/crypto/dca.h>
40 
41 
42 static void dca_rsaverifydone(dca_request_t *, int);
43 static void dca_rsadone(dca_request_t *, int);
44 
45 /* Exported function prototypes */
46 int dca_rsastart(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
47     crypto_req_handle_t, int);
48 int dca_rsainit(crypto_ctx_t *, crypto_mechanism_t *, crypto_key_t *, int);
49 void dca_rsactxfree(void *);
50 int dca_rsaatomic(crypto_provider_handle_t, crypto_session_id_t,
51     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *,
52     int, crypto_req_handle_t, int);
53 
54 /* Local function prototypes */
55 static int dca_pkcs1_padding(dca_t *dca, caddr_t buf, int flen, int tlen,
56     int private);
57 static int dca_pkcs1_unpadding(char *buf, int *tlen, int flen, int mode);
58 static int dca_x509_padding(caddr_t buf, int flen, int tlen);
59 static int dca_x509_unpadding(char *buf, int tlen, int flen, int mode);
60 static int decrypt_error_code(int mode, int decrypt, int verify, int def);
61 
62 
63 int dca_rsastart(crypto_ctx_t *ctx, crypto_data_t *in, crypto_data_t *out,
64     crypto_req_handle_t req, int mode)
65 {
66 	dca_request_t		*reqp = ctx->cc_provider_private;
67 	dca_t			*dca = ctx->cc_provider;
68 	caddr_t			daddr;
69 	int			rv = CRYPTO_QUEUED;
70 	int			len;
71 
72 	/*
73 	 * In-place operations (in == out) are indicated by having a
74 	 * NULL output. In this case set the out to point to the in.
75 	 * Note that this only works for CKM_RSA_X_509 without any padding
76 	 */
77 	if (!out) {
78 		DBG(dca, DWARN, "Using inline since output buffer is NULL.");
79 		out = in;
80 	}
81 
82 	/* We don't support non-contiguous buffers for RSA */
83 	if (dca_sgcheck(dca, in, DCA_SG_CONTIG) ||
84 	    dca_sgcheck(dca, out, DCA_SG_CONTIG)) {
85 		rv = CRYPTO_NOT_SUPPORTED;
86 		goto errout;
87 	}
88 
89 	len = dca_length(in);
90 
91 	/* Extracting the key attributes is now done in dca_rsainit(). */
92 	if (mode == DCA_RSA_ENC || mode == DCA_RSA_SIGN ||
93 	    mode == DCA_RSA_SIGNR) {
94 		/*
95 		 * Return length needed to store the output.
96 		 * For sign, sign-recover, and encrypt, the output buffer
97 		 * should not be smaller than modlen since PKCS or X_509
98 		 * padding will be applied
99 		 */
100 		if (dca_length(out) < reqp->dr_ctx.modlen) {
101 			DBG(dca, DWARN,
102 			    "dca_rsastart: output buffer too short (%d < %d)",
103 			    dca_length(out), reqp->dr_ctx.modlen);
104 			out->cd_length = reqp->dr_ctx.modlen;
105 			rv = CRYPTO_BUFFER_TOO_SMALL;
106 			goto errout;
107 		}
108 	}
109 	if (out != in && out->cd_length > reqp->dr_ctx.modlen)
110 		out->cd_length = reqp->dr_ctx.modlen;
111 
112 	/* The input length should not be bigger than the modulus */
113 	if (len > reqp->dr_ctx.modlen) {
114 		rv = decrypt_error_code(mode, CRYPTO_ENCRYPTED_DATA_LEN_RANGE,
115 		    CRYPTO_SIGNATURE_LEN_RANGE, CRYPTO_DATA_LEN_RANGE);
116 		goto errout;
117 	}
118 
119 	/*
120 	 * For decryption, verify, and verifyRecover, the input length should
121 	 * not be less than the modulus
122 	 */
123 	if (len < reqp->dr_ctx.modlen && (mode == DCA_RSA_DEC ||
124 	    mode == DCA_RSA_VRFY || mode == DCA_RSA_VRFYR)) {
125 		rv = decrypt_error_code(mode, CRYPTO_ENCRYPTED_DATA_LEN_RANGE,
126 		    CRYPTO_SIGNATURE_LEN_RANGE, CRYPTO_DATA_LEN_RANGE);
127 		goto errout;
128 	}
129 
130 	/*
131 	 * For decryption and verifyRecover, the output buffer should not
132 	 * be less than the modulus
133 	 */
134 	if (out->cd_length < reqp->dr_ctx.modlen && (mode == DCA_RSA_DEC ||
135 	    mode == DCA_RSA_VRFYR) &&
136 	    reqp->dr_ctx.ctx_cm_type == RSA_X_509_MECH_INFO_TYPE) {
137 		out->cd_length = reqp->dr_ctx.modlen;
138 		rv = CRYPTO_BUFFER_TOO_SMALL;
139 		goto errout;
140 	}
141 
142 	/* For decrypt and verify, the input should not be less than output */
143 	if (out && len < out->cd_length) {
144 		if ((rv = decrypt_error_code(mode,
145 		    CRYPTO_ENCRYPTED_DATA_LEN_RANGE,
146 		    CRYPTO_SIGNATURE_LEN_RANGE, CRYPTO_SUCCESS)) !=
147 		    CRYPTO_SUCCESS)
148 			goto errout;
149 	}
150 
151 	if ((daddr = dca_bufdaddr(in)) == NULL && len > 0) {
152 		rv = CRYPTO_ARGUMENTS_BAD;
153 		goto errout;
154 	}
155 
156 	if (dca_numcmp(daddr, len, (char *)reqp->dr_ctx.mod,
157 	    reqp->dr_ctx.modlen) > 0) {
158 		DBG(dca, DWARN,
159 		    "dca_rsastart: input larger (numerically) than modulus!");
160 		rv = decrypt_error_code(mode, CRYPTO_ENCRYPTED_DATA_INVALID,
161 		    CRYPTO_SIGNATURE_INVALID, CRYPTO_DATA_INVALID);
162 		goto errout;
163 	}
164 
165 	reqp->dr_byte_stat = -1;
166 	reqp->dr_in = in;
167 	reqp->dr_out = out;
168 	reqp->dr_kcf_req = req;
169 	if (mode == DCA_RSA_VRFY)
170 		reqp->dr_callback = dca_rsaverifydone;
171 	else
172 		reqp->dr_callback = dca_rsadone;
173 
174 	dca_reverse(daddr, reqp->dr_ibuf_kaddr, len, reqp->dr_pkt_length);
175 	if (mode == DCA_RSA_ENC || mode == DCA_RSA_SIGN ||
176 	    mode == DCA_RSA_SIGNR) {
177 		/*
178 		 * Needs to pad appropriately for encrypt, sign, and
179 		 * sign_recover
180 		 */
181 		if (reqp->dr_ctx.ctx_cm_type == RSA_PKCS_MECH_INFO_TYPE) {
182 			if ((rv = dca_pkcs1_padding(dca, reqp->dr_ibuf_kaddr,
183 			    len, reqp->dr_ctx.modlen, reqp->dr_ctx.pqfix)) !=
184 			    CRYPTO_QUEUED)
185 				goto errout;
186 		} else if (reqp->dr_ctx.ctx_cm_type ==
187 		    RSA_X_509_MECH_INFO_TYPE) {
188 			if ((rv = dca_x509_padding(reqp->dr_ibuf_kaddr,
189 			    len, reqp->dr_pkt_length)) != CRYPTO_QUEUED)
190 				goto errout;
191 		}
192 	}
193 	reqp->dr_ctx.mode = mode;
194 
195 	/*
196 	 * Since the max RSA input size is 256 bytes (2048 bits), the firstx
197 	 * page (at least 4096 bytes) in the pre-mapped buffer is large enough.
198 	 * Therefore, we use this first page for RSA.
199 	 */
200 	reqp->dr_in_paddr = reqp->dr_ibuf_head.dc_buffer_paddr;
201 	reqp->dr_in_next = 0;
202 	reqp->dr_in_len = reqp->dr_pkt_length;
203 	reqp->dr_out_paddr = reqp->dr_obuf_head.dc_buffer_paddr;
204 	reqp->dr_out_next = 0;
205 	reqp->dr_out_len = reqp->dr_pkt_length;
206 
207 	/* schedule the work by doing a submit */
208 	rv = dca_start(dca, reqp, MCR2, 1);
209 
210 
211 errout:
212 	if (rv != CRYPTO_QUEUED && rv != CRYPTO_BUFFER_TOO_SMALL)
213 		(void) dca_free_context(ctx);
214 
215 	return (rv);
216 }
217 
218 void
219 dca_rsadone(dca_request_t *reqp, int errno)
220 {
221 	if (errno == CRYPTO_SUCCESS) {
222 		int	outsz = reqp->dr_out->cd_length;
223 		caddr_t	daddr;
224 
225 		(void) ddi_dma_sync(reqp->dr_obuf_dmah, 0, reqp->dr_out_len,
226 		    DDI_DMA_SYNC_FORKERNEL);
227 		if (dca_check_dma_handle(reqp->dr_dca, reqp->dr_obuf_dmah,
228 		    DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
229 			reqp->destroy = TRUE;
230 			errno = CRYPTO_DEVICE_ERROR;
231 			goto errout;
232 		}
233 
234 		if (reqp->dr_ctx.mode == DCA_RSA_DEC ||
235 		    reqp->dr_ctx.mode == DCA_RSA_VRFY ||
236 		    reqp->dr_ctx.mode == DCA_RSA_VRFYR) {
237 			/*
238 			 * Needs to unpad appropriately for decrypt, verify,
239 			 * and verify_recover
240 			 */
241 			if (reqp->dr_ctx.ctx_cm_type ==
242 			    RSA_PKCS_MECH_INFO_TYPE) {
243 				errno = dca_pkcs1_unpadding(
244 				    reqp->dr_obuf_kaddr, &outsz,
245 				    reqp->dr_ctx.modlen, reqp->dr_ctx.mode);
246 
247 				/* check for bad data errors */
248 				if (errno != CRYPTO_SUCCESS &&
249 				    errno != CRYPTO_BUFFER_TOO_SMALL) {
250 					goto errout;
251 				}
252 				if (dca_bufdaddr(reqp->dr_out) == NULL) {
253 					errno = CRYPTO_BUFFER_TOO_SMALL;
254 				}
255 				if (errno == CRYPTO_BUFFER_TOO_SMALL) {
256 					reqp->dr_out->cd_length = outsz;
257 					goto errout;
258 				}
259 				/* Reset the output data length */
260 				reqp->dr_out->cd_length = outsz;
261 			} else if (reqp->dr_ctx.ctx_cm_type ==
262 			    RSA_X_509_MECH_INFO_TYPE) {
263 				if ((errno = dca_x509_unpadding(
264 				    reqp->dr_obuf_kaddr, outsz,
265 				    reqp->dr_pkt_length, reqp->dr_ctx.mode)) !=
266 				    CRYPTO_SUCCESS)
267 					goto errout;
268 			}
269 		}
270 
271 		if ((daddr = dca_bufdaddr(reqp->dr_out)) == NULL) {
272 			DBG(reqp->dr_dca, DINTR,
273 			    "dca_rsadone: reqp->dr_out is bad");
274 			errno = CRYPTO_ARGUMENTS_BAD;
275 			goto errout;
276 		}
277 		/*
278 		 * Note that there may be some number of null bytes
279 		 * at the end of the source (result), but we don't care
280 		 * about them -- they are place holders only and are
281 		 * truncated here.
282 		 */
283 		dca_reverse(reqp->dr_obuf_kaddr, daddr, outsz, outsz);
284 	}
285 errout:
286 	ASSERT(reqp->dr_kcf_req != NULL);
287 
288 	/* notify framework that request is completed */
289 	crypto_op_notification(reqp->dr_kcf_req, errno);
290 	DBG(reqp->dr_dca, DINTR,
291 	    "dca_rsadone: returning 0x%x to the kef via crypto_op_notification",
292 	    errno);
293 
294 	/*
295 	 * For non-atomic operations, reqp will be freed in the kCF
296 	 * callback function since it may be needed again if
297 	 * CRYPTO_BUFFER_TOO_SMALL is returned to kCF
298 	 */
299 	if (reqp->dr_ctx.atomic) {
300 		crypto_ctx_t ctx;
301 		ctx.cc_provider_private = reqp;
302 		dca_rsactxfree(&ctx);
303 	}
304 }
305 
306 void
307 dca_rsaverifydone(dca_request_t *reqp, int errno)
308 {
309 	if (errno == CRYPTO_SUCCESS) {
310 		char	scratch[RSA_MAX_KEY_LEN];
311 		int	outsz = reqp->dr_out->cd_length;
312 		caddr_t	daddr;
313 
314 		/*
315 		 * ASSUMPTION: the signature length was already
316 		 * checked on the way in, and it is a valid length.
317 		 */
318 		(void) ddi_dma_sync(reqp->dr_obuf_dmah, 0, outsz,
319 		    DDI_DMA_SYNC_FORKERNEL);
320 		if (dca_check_dma_handle(reqp->dr_dca, reqp->dr_obuf_dmah,
321 		    DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
322 			reqp->destroy = TRUE;
323 			errno = CRYPTO_DEVICE_ERROR;
324 			goto errout;
325 		}
326 
327 		if (reqp->dr_ctx.mode == DCA_RSA_DEC ||
328 		    reqp->dr_ctx.mode == DCA_RSA_VRFY ||
329 		    reqp->dr_ctx.mode == DCA_RSA_VRFYR) {
330 			/*
331 			 * Needs to unpad appropriately for decrypt, verify,
332 			 * and verify_recover
333 			 */
334 			if (reqp->dr_ctx.ctx_cm_type ==
335 			    RSA_PKCS_MECH_INFO_TYPE) {
336 				errno = dca_pkcs1_unpadding(
337 				    reqp->dr_obuf_kaddr, &outsz,
338 				    reqp->dr_ctx.modlen, reqp->dr_ctx.mode);
339 
340 				/* check for bad data errors */
341 				if (errno != CRYPTO_SUCCESS &&
342 				    errno != CRYPTO_BUFFER_TOO_SMALL) {
343 					goto errout;
344 				}
345 				if (dca_bufdaddr(reqp->dr_out) == NULL) {
346 					errno = CRYPTO_BUFFER_TOO_SMALL;
347 				}
348 				if (errno == CRYPTO_BUFFER_TOO_SMALL) {
349 					reqp->dr_out->cd_length = outsz;
350 					goto errout;
351 				}
352 				/* Reset the output data length */
353 				reqp->dr_out->cd_length = outsz;
354 			} else if (reqp->dr_ctx.ctx_cm_type ==
355 			    RSA_X_509_MECH_INFO_TYPE) {
356 				if ((errno = dca_x509_unpadding(
357 				    reqp->dr_obuf_kaddr, outsz,
358 				    reqp->dr_pkt_length, reqp->dr_ctx.mode)) !=
359 				    CRYPTO_SUCCESS)
360 					goto errout;
361 			}
362 		}
363 
364 		dca_reverse(reqp->dr_obuf_kaddr, scratch, outsz, outsz);
365 
366 		if ((daddr = dca_bufdaddr(reqp->dr_out)) == NULL) {
367 			errno = CRYPTO_ARGUMENTS_BAD;
368 			goto errout;
369 		}
370 		if (dca_numcmp(daddr, reqp->dr_out->cd_length, scratch,
371 		    outsz) != 0) {
372 			/* VERIFY FAILED */
373 			errno = CRYPTO_SIGNATURE_INVALID;
374 		}
375 	}
376 errout:
377 	ASSERT(reqp->dr_kcf_req != NULL);
378 
379 	/* notify framework that request is completed */
380 	crypto_op_notification(reqp->dr_kcf_req, errno);
381 	DBG(reqp->dr_dca, DINTR,
382 	    "dca_rsaverifydone: rtn 0x%x to the kef via crypto_op_notification",
383 	    errno);
384 
385 	/*
386 	 * For non-atomic operations, reqp will be freed in the kCF
387 	 * callback function since it may be needed again if
388 	 * CRYPTO_BUFFER_TOO_SMALL is returned to kCF
389 	 */
390 	if (reqp->dr_ctx.atomic) {
391 		crypto_ctx_t ctx;
392 		ctx.cc_provider_private = reqp;
393 		dca_rsactxfree(&ctx);
394 	}
395 }
396 
397 /*
398  * Setup either a public or a private RSA key for subsequent uses
399  */
400 int
401 dca_rsainit(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
402     crypto_key_t *key, int kmflag)
403 {
404 	crypto_object_attribute_t	*attr;
405 	unsigned			expname = 0;
406 	void				*attrdata;
407 	int rv;
408 
409 	uchar_t			*exp;
410 	uchar_t			*p;
411 	uchar_t			*q;
412 	uchar_t			*dp;
413 	uchar_t			*dq;
414 	uchar_t			*pinv;
415 
416 	unsigned		explen = 0;
417 	unsigned		plen = 0;
418 	unsigned		qlen = 0;
419 	unsigned		dplen = 0;
420 	unsigned		dqlen = 0;
421 	unsigned		pinvlen = 0;
422 
423 	unsigned		modbits, expbits, pbits, qbits;
424 	unsigned		modfix, expfix, pqfix = 0;
425 	uint16_t		ctxlen;
426 	caddr_t			kaddr;
427 	dca_request_t		*reqp = NULL;
428 	dca_t			*dca = (dca_t *)ctx->cc_provider;
429 
430 	DBG(NULL, DENTRY, "dca_rsainit: start");
431 
432 	if ((reqp = dca_getreq(dca, MCR2, 1)) == NULL) {
433 		DBG(NULL, DWARN,
434 		    "dca_rsainit: unable to allocate request for RSA");
435 		rv = CRYPTO_HOST_MEMORY;
436 		goto errout;
437 	}
438 
439 	reqp->dr_ctx.ctx_cm_type = mechanism->cm_type;
440 	ctx->cc_provider_private = reqp;
441 
442 	/*
443 	 * Key type can be either RAW, or REFERENCE, or ATTR_LIST (VALUE).
444 	 * Only ATTR_LIST is supported on Deimos for RSA.
445 	 */
446 	if ((attr = dca_get_key_attr(key)) == NULL) {
447 		DBG(NULL, DWARN, "dca_rsainit: key attributes missing");
448 		rv = CRYPTO_KEY_TYPE_INCONSISTENT;
449 		goto errout;
450 	}
451 
452 	if (dca_find_attribute(attr, key->ck_count, CKA_PUBLIC_EXPONENT))
453 		expname = CKA_PUBLIC_EXPONENT;
454 
455 	/*
456 	 * RSA public key has only public exponent. RSA private key must have
457 	 * private exponent. However, it may also have public exponent.
458 	 * Thus, the existance of a private exponent indicates a private key.
459 	 */
460 	if (dca_find_attribute(attr, key->ck_count, CKA_PRIVATE_EXPONENT))
461 		expname = CKA_PRIVATE_EXPONENT;
462 
463 	if (!expname) {
464 		DBG(NULL, DWARN, "dca_rsainit: no exponent in key");
465 		rv = CRYPTO_ARGUMENTS_BAD;
466 		goto errout;
467 	}
468 
469 	/* Modulus */
470 	if ((rv = dca_attr_lookup_uint8_array(attr, key->ck_count, CKA_MODULUS,
471 	    &attrdata, &(reqp->dr_ctx.modlen))) != CRYPTO_SUCCESS) {
472 		DBG(NULL, DWARN, "dca_rsainit: failed to retrieve modulus");
473 		goto errout;
474 	}
475 	if ((reqp->dr_ctx.modlen == 0) ||
476 	    (reqp->dr_ctx.modlen > RSA_MAX_KEY_LEN)) {
477 		DBG(NULL, DWARN, "dca_rsainit: bad modulus size");
478 		rv = CRYPTO_ARGUMENTS_BAD;
479 		goto errout;
480 	}
481 	if ((reqp->dr_ctx.mod = kmem_alloc(reqp->dr_ctx.modlen, kmflag)) ==
482 	    NULL) {
483 		rv = CRYPTO_HOST_MEMORY;
484 		goto errout;
485 	}
486 	bcopy(attrdata, reqp->dr_ctx.mod, reqp->dr_ctx.modlen);
487 
488 	/* Exponent */
489 	if ((rv = dca_attr_lookup_uint8_array(attr, key->ck_count, expname,
490 	    (void **) &exp, &explen)) != CRYPTO_SUCCESS) {
491 		DBG(NULL, DWARN, "dca_rsainit: failed to retrieve exponent");
492 		goto errout;
493 	}
494 	if ((explen == 0) || (explen > RSA_MAX_KEY_LEN)) {
495 		DBG(NULL, DWARN, "dca_rsainit: bad exponent size");
496 		rv = CRYPTO_ARGUMENTS_BAD;
497 		goto errout;
498 	}
499 
500 	/* Lookup private attributes */
501 	if (expname == CKA_PRIVATE_EXPONENT) {
502 		/* Prime 1 */
503 		(void) dca_attr_lookup_uint8_array(attr, key->ck_count,
504 		    CKA_PRIME_1, (void **)&q, &qlen);
505 
506 		/* Prime 2 */
507 		(void) dca_attr_lookup_uint8_array(attr, key->ck_count,
508 		    CKA_PRIME_2, (void **)&p, &plen);
509 
510 		/* Exponent 1 */
511 		(void) dca_attr_lookup_uint8_array(attr, key->ck_count,
512 		    CKA_EXPONENT_1, (void **)&dq, &dqlen);
513 
514 		/* Exponent 2 */
515 		(void) dca_attr_lookup_uint8_array(attr, key->ck_count,
516 		    CKA_EXPONENT_2, (void **)&dp, &dplen);
517 
518 		/* Coefficient */
519 		(void) dca_attr_lookup_uint8_array(attr, key->ck_count,
520 		    CKA_COEFFICIENT, (void **)&pinv, &pinvlen);
521 	}
522 
523 	modbits = dca_bitlen(reqp->dr_ctx.mod, reqp->dr_ctx.modlen);
524 	expbits = dca_bitlen(exp, explen);
525 
526 	if ((modfix = dca_padfull(modbits)) == 0) {
527 		DBG(NULL, DWARN, "dca_rsainit: modulus too long");
528 		rv = CRYPTO_KEY_SIZE_RANGE;
529 		goto errout;
530 	}
531 	expfix =  ROUNDUP(explen, sizeof (uint32_t));
532 
533 	if (plen && qlen && dplen && dqlen && pinvlen) {
534 		unsigned pfix, qfix;
535 		qbits = dca_bitlen(q, qlen);
536 		pbits = dca_bitlen(p, plen);
537 		qfix = dca_padhalf(qbits);
538 		pfix = dca_padhalf(pbits);
539 		if (pfix & qfix)
540 			pqfix = max(pfix, qfix);
541 	}
542 
543 	if (pqfix) {
544 		reqp->dr_job_stat = DS_RSAPRIVATE;
545 		reqp->dr_pkt_length = 2 * pqfix;
546 	} else {
547 		reqp->dr_job_stat = DS_RSAPUBLIC;
548 		reqp->dr_pkt_length = modfix;
549 	}
550 
551 	if (pqfix) {
552 		/*
553 		 * NOTE: chip's notion of p vs. q is reversed from
554 		 * PKCS#11.  We use the chip's notion in our variable
555 		 * naming.
556 		 */
557 		ctxlen = 8 + pqfix * 5;
558 
559 		/* write out the context structure */
560 		PUTCTX16(reqp, CTX_CMD, CMD_RSAPRIVATE);
561 		PUTCTX16(reqp, CTX_LENGTH, ctxlen);
562 		/* exponent and modulus length in bits!!! */
563 		PUTCTX16(reqp, CTX_RSAQLEN, qbits);
564 		PUTCTX16(reqp, CTX_RSAPLEN, pbits);
565 
566 		kaddr = reqp->dr_ctx_kaddr + CTX_RSABIGNUMS;
567 
568 		/* store the bignums */
569 		dca_reverse(p, kaddr, plen, pqfix);
570 		kaddr += pqfix;
571 
572 		dca_reverse(q, kaddr, qlen, pqfix);
573 		kaddr += pqfix;
574 
575 		dca_reverse(dp, kaddr, dplen, pqfix);
576 		kaddr += pqfix;
577 
578 		dca_reverse(dq, kaddr, dqlen, pqfix);
579 		kaddr += pqfix;
580 
581 		dca_reverse(pinv, kaddr, pinvlen, pqfix);
582 		kaddr += pqfix;
583 	} else {
584 		ctxlen = 8 + modfix + expfix;
585 		/* write out the context structure */
586 		PUTCTX16(reqp, CTX_CMD, CMD_RSAPUBLIC);
587 		PUTCTX16(reqp, CTX_LENGTH, (uint16_t)ctxlen);
588 		/* exponent and modulus length in bits!!! */
589 		PUTCTX16(reqp, CTX_RSAEXPLEN, expbits);
590 		PUTCTX16(reqp, CTX_RSAMODLEN, modbits);
591 
592 		kaddr = reqp->dr_ctx_kaddr + CTX_RSABIGNUMS;
593 
594 		/* store the bignums */
595 		dca_reverse(reqp->dr_ctx.mod, kaddr, reqp->dr_ctx.modlen,
596 		    modfix);
597 		kaddr += modfix;
598 
599 		dca_reverse(exp, kaddr, explen, expfix);
600 		kaddr += expfix;
601 	}
602 
603 	reqp->dr_ctx.pqfix = pqfix;
604 
605 errout:
606 	if (rv != CRYPTO_SUCCESS)
607 		dca_rsactxfree(ctx);
608 
609 	return (rv);
610 }
611 
612 void
613 dca_rsactxfree(void *arg)
614 {
615 	crypto_ctx_t	*ctx = (crypto_ctx_t *)arg;
616 	dca_request_t	*reqp = ctx->cc_provider_private;
617 
618 	if (reqp == NULL)
619 		return;
620 
621 	if (reqp->dr_ctx.mod)
622 		kmem_free(reqp->dr_ctx.mod, reqp->dr_ctx.modlen);
623 
624 	reqp->dr_ctx.mode = 0;
625 	reqp->dr_ctx.ctx_cm_type = 0;
626 	reqp->dr_ctx.mod = NULL;
627 	reqp->dr_ctx.modlen = 0;
628 	reqp->dr_ctx.pqfix = 0;
629 	reqp->dr_ctx.atomic = 0;
630 
631 	if (reqp->destroy)
632 		dca_destroyreq(reqp);
633 	else
634 		dca_freereq(reqp);
635 
636 	ctx->cc_provider_private = NULL;
637 }
638 
639 int
640 dca_rsaatomic(crypto_provider_handle_t provider,
641     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
642     crypto_key_t *key, crypto_data_t *input, crypto_data_t *output,
643     int kmflag, crypto_req_handle_t req, int mode)
644 {
645 	crypto_ctx_t	ctx;	/* on the stack */
646 	int		rv;
647 
648 	ctx.cc_provider = provider;
649 	ctx.cc_session = session_id;
650 
651 	rv = dca_rsainit(&ctx, mechanism, key, kmflag);
652 	if (rv != CRYPTO_SUCCESS) {
653 		DBG(NULL, DWARN, "dca_rsaatomic: dca_rsainit() failed");
654 		/* The content of ctx should have been freed already */
655 		return (rv);
656 	}
657 
658 	/*
659 	 * Set the atomic flag so that the hardware callback function
660 	 * will free the context.
661 	 */
662 	((dca_request_t *)ctx.cc_provider_private)->dr_ctx.atomic = 1;
663 
664 	rv = dca_rsastart(&ctx, input, output, req, mode);
665 
666 	/*
667 	 * The context will be freed in the hardware callback function if it
668 	 * is queued
669 	 */
670 	if (rv != CRYPTO_QUEUED)
671 		dca_rsactxfree(&ctx);
672 
673 	return (rv);
674 }
675 
676 
677 /*
678  * For RSA_PKCS padding and unpadding:
679  * 1. The minimum padding is 11 bytes.
680  * 2. The first and the last bytes must 0.
681  * 3. The second byte is 1 for private and 2 for public keys.
682  * 4. Pad with 0xff for private and non-zero random for public keys.
683  */
684 static int
685 dca_pkcs1_padding(dca_t *dca, caddr_t buf, int flen, int tlen, int private)
686 {
687 	int i;
688 
689 	DBG(NULL, DENTRY,
690 	    "dca_pkcs1_padding: tlen: %d, flen: %d: private: %d\n",
691 	    tlen, flen, private);
692 
693 	if (flen > tlen - 11)
694 		return (CRYPTO_DATA_LEN_RANGE);
695 
696 	if (private) {
697 		/* Padding for private encrypt */
698 		buf[flen] = '\0';
699 		for (i = flen + 1; i < tlen - 2; i++) {
700 			buf[i] = (unsigned char) 0xff;
701 		}
702 		buf[tlen - 2] = 1;
703 		buf[tlen - 1] = 0;
704 	} else {
705 		/* Padding for public encrypt */
706 		buf[flen] = '\0';
707 
708 		if (dca_random_buffer(dca, &buf[flen+1], tlen - flen - 3) !=
709 		    CRYPTO_SUCCESS)
710 			return (CRYPTO_RANDOM_NO_RNG);
711 
712 		buf[tlen - 2] = 2;
713 		buf[tlen - 1] = 0;
714 	}
715 
716 	return (CRYPTO_QUEUED);
717 }
718 
719 static int
720 dca_pkcs1_unpadding(char *buf, int *tlen, int flen, int mode)
721 {
722 	int i;
723 	const unsigned char *p;
724 	unsigned char type;
725 
726 	DBG(NULL, DENTRY, "dca_pkcs1_unpadding: tlen: %d, flen: %d\n",
727 	    *tlen, flen);
728 
729 	p = (unsigned char *) buf + (flen-1);
730 	if (*(p--) != 0)
731 		return decrypt_error_code(mode, CRYPTO_ENCRYPTED_DATA_INVALID,
732 		    CRYPTO_SIGNATURE_INVALID, CRYPTO_DATA_INVALID);
733 
734 	/* It is ok if the data length is 0 after removing the padding */
735 	type = *(p--);
736 	if (type == 01) {
737 		for (i = flen - 3; i >= 0; i--) {
738 			if (*p != 0xff) {
739 				if (*p == '\0') {
740 					p--;
741 					break;
742 				} else {
743 					return decrypt_error_code(mode,
744 					    CRYPTO_ENCRYPTED_DATA_INVALID,
745 					    CRYPTO_SIGNATURE_INVALID,
746 					    CRYPTO_DATA_INVALID);
747 				}
748 			}
749 			p--;
750 		}
751 	} else if (type == 02) {
752 		for (i = flen - 3; i >= 0; i--) {
753 			if (*p == '\0') {
754 				p--;
755 				break;
756 			}
757 			p--;
758 		}
759 	} else {
760 		return decrypt_error_code(mode, CRYPTO_ENCRYPTED_DATA_INVALID,
761 		    CRYPTO_SIGNATURE_INVALID, CRYPTO_DATA_INVALID);
762 	}
763 
764 	/* i < 0 means did not find the end of the padding */
765 	if (i < 0)
766 		return decrypt_error_code(mode, CRYPTO_ENCRYPTED_DATA_INVALID,
767 		    CRYPTO_SIGNATURE_INVALID, CRYPTO_DATA_INVALID);
768 
769 	if (i > *tlen) {
770 		*tlen = i;
771 		return (CRYPTO_BUFFER_TOO_SMALL);
772 	}
773 
774 	if (flen - i < 11)
775 		return decrypt_error_code(mode,
776 		    CRYPTO_ENCRYPTED_DATA_LEN_RANGE,
777 		    CRYPTO_SIGNATURE_LEN_RANGE, CRYPTO_DATA_LEN_RANGE);
778 
779 	/* Return the unpadded length to the caller */
780 	*tlen = i;
781 
782 	return (CRYPTO_SUCCESS);
783 }
784 
785 /*
786  * For RSA_X_509 padding and unpadding, pad all 0s before actual data.
787  * Note that the data will be in reverse order.
788  */
789 static int
790 dca_x509_padding(caddr_t buf, int flen, int tlen)
791 {
792 	DBG(NULL, DENTRY, "dca_x509_padding: tlen: %d, flen: %d\n",
793 	    tlen, flen);
794 
795 	bzero(buf+tlen, tlen - flen);
796 
797 	return (CRYPTO_QUEUED);
798 }
799 
800 /* ARGSUSED */
801 static int
802 dca_x509_unpadding(char *buf, int tlen, int flen, int mode)
803 {
804 	int i;
805 	const unsigned char *p;
806 
807 	DBG(NULL, DENTRY, "dca_x509_unpadding: tlen: %d, flen: %d\n",
808 	    tlen, flen);
809 
810 	p = (unsigned char *) buf + flen;
811 	for (i = tlen; i < flen; i++) {
812 		if (*(--p) != 0)
813 			return (CRYPTO_SIGNATURE_INVALID);
814 	}
815 
816 	return (CRYPTO_SUCCESS);
817 }
818 
819 static int decrypt_error_code(int mode, int decrypt, int verify, int def)
820 {
821 	switch (mode) {
822 	case DCA_RSA_DEC:
823 		return (decrypt);
824 	case DCA_RSA_VRFY:
825 	case DCA_RSA_VRFYR:
826 		return (verify);
827 	default:
828 		return (def);
829 	}
830 }
831