xref: /illumos-gate/usr/src/common/crypto/padding/pkcs1.c (revision 45ede40b2394db7967e59f19288fae9b62efd4aa)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 
26 /*
27  * This file contains padding helper routines common to
28  * the PKCS11 soft token code and the kernel crypto code.
29  */
30 
31 #include <sys/types.h>
32 #include "padding.h"
33 
34 #ifdef _KERNEL
35 #include <sys/param.h>
36 #else
37 #include <strings.h>
38 #include <cryptoutil.h>
39 #endif
40 
41 /*
42  * To create a block type "02" encryption block for RSA PKCS encryption
43  * process.
44  *
45  * This is EME-PKCS1-v1_5 encoding as described in RSA PKCS#1.
46  *
47  * The RSA PKCS Padding before encryption is in the following format:
48  * +----+----+--------------------+----+-----------------------------+
49  * |0x00|0x02| 8 bytes or more RN |0x00|       DATA                  |
50  * +----+----+--------------------+----+-----------------------------+
51  *
52  *
53  * To create a block type "01" block for RSA PKCS signature process.
54  *
55  * This EMSA-PKCS1-1_5 encoding as decribed in RSA PKCS#1.
56  *
57  * The RSA PKCS Padding before Signing is in the following format:
58  * +----+----+----------------------+----+-----------------------------+
59  * |0x00|0x01| 8 bytes of more 0xFF |0x00|          DATA               |
60  * +----+----+----------------------+----+-----------------------------+
61  *
62  */
63 int
64 pkcs1_encode(int method, uint8_t *databuf, size_t datalen, uint8_t *padbuf,
65     size_t padbuflen)
66 {
67 	size_t	padlen;
68 	int	rv;
69 
70 	padlen = padbuflen - datalen;
71 	if (padlen < MIN_PKCS1_PADLEN) {
72 		return (CKR_DATA_LEN_RANGE);
73 	}
74 
75 	rv = 0;
76 
77 	padbuf[0] = 0x00;
78 	padbuf[1] = (method == PKCS1_ENCRYPT) ? 0x02 : 0x01;
79 
80 	if (method == PKCS1_ENCRYPT) {
81 #ifdef _KERNEL
82 		rv = knzero_random_generator(padbuf + 2, padlen - 3);
83 #else
84 		rv = (pkcs11_get_nzero_urandom(padbuf + 2, padlen - 3) < 0) ?
85 		    CKR_DEVICE_ERROR : 0;
86 #endif
87 	} else if (method == PKCS1_SIGN) {
88 #ifdef _KERNEL
89 		kmemset(padbuf + 2, 0xFF, padlen - 3);
90 #else
91 		(void) memset(padbuf + 2, 0xFF, padlen - 3);
92 #endif
93 	}
94 
95 	if (rv != 0) {
96 		return (rv);
97 	}
98 
99 	padbuf[padlen - 1] = 0x00;
100 
101 	bcopy(databuf, padbuf + padlen, datalen);
102 
103 	return (0);
104 }
105 
106 /*
107  * The RSA PKCS Padding in the following format:
108  * +----+----+-------------------------+----+------------------------+
109  * |0x00| BT | 8 bytes or more padding |0x00|       DATA             |
110  * +----+----+-+++++-------------------+----+------------------------+
111  * where BT is block type: 0x02 for encrypt/decrypt, 0x01 for sign/verify
112  *
113  * 'padbuf' points to the recovered message.  Strip off the padding and
114  * validate it as much as possible.  'plen' is changed to hold the actual
115  * data length.
116  */
117 int
118 pkcs1_decode(int method, uint8_t *padbuf, size_t *plen)
119 {
120 	int	rv = ((method == PKCS1_DECRYPT) ? CKR_ENCRYPTED_DATA_INVALID :
121 	    CKR_SIGNATURE_INVALID);
122 	int	i;
123 
124 	/* Check to see if the recovered data is padded is 0x0002 or 0x0001. */
125 	if (padbuf[0] != 0x00 || padbuf[1] != (method == PKCS1_DECRYPT ?
126 	    0x02 : 0x01)) {
127 		return (rv);
128 	}
129 
130 	/* Remove all the random bits up to 0x00 (= NULL char) */
131 	for (i = 2; (*plen - i) > 0; i++) {
132 		if (padbuf[i] == 0x00) {
133 			i++;
134 			if (i < MIN_PKCS1_PADLEN) {
135 				return (rv);
136 			}
137 			*plen -= i;
138 
139 			return (0);
140 		} else if (method == PKCS1_VERIFY && padbuf[i] != 0xFF) {
141 			return (rv);
142 		}
143 	}
144 
145 	return (rv);
146 }
147