xref: /illumos-gate/usr/src/common/crypto/padding/pkcs7.c (revision 4f364e7c95ee7fd9d5bbeddc1940e92405bb0e72)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 
26 /*
27  * This file contains padding helper routines common to
28  * the PKCS11 soft token code and the kernel crypto code.
29  */
30 
31 #include <sys/types.h>
32 #include "padding.h"
33 
34 #ifdef _KERNEL
35 #include <sys/param.h>
36 #else
37 #include <strings.h>
38 #include <cryptoutil.h>
39 #endif
40 
41 /*
42  * This is padding as decribed in Section 10.3 of RSA PKCS#7.
43  *
44  * The RSA PKCS Padding is in the following format:
45  * +-----------------------------+----+-------------+
46  * |       DATA                  |0x0k|0x0k|...|0x0k|
47  * +-----------------------------+----+----+---+----+
48  * where 0x0k is if data_len mod multiple = multiple - k
49  * and multiple < 256 and 1 <= k <= multiple
50  *
51  * If databuf is non NULL, padbuf must be large enough
52  * to contain both databuf and the padding.  databuf and
53  * padbuf may be the same buffer.
54  * databuf:
55  * +-----------------------------+
56  * |       DATA                  |
57  * +-----------------------------+
58  *                           datalen
59  * padbuf:
60  * +-----------------------------+----+-------------+
61  * |       DATA                  |0x0k|0x0k|...|0x0k|
62  * +-----------------------------+----+----+---+----+
63  *                           datalen          padbuflen
64  *
65  * If databuf is NULL, padbuf only needs to be large
66  * enough for the padding, and datalen must still be
67  * provided to compute the padding value:
68  *				 padbuf:
69  *                               +----+-------------+
70  *                               |0x0k|0x0k|...|0x0k|
71  *                               +----+----+---+----+
72  *                           datalen           padbuflen
73  */
74 int
75 pkcs7_encode(uint8_t *databuf, size_t datalen, uint8_t *padbuf,
76     size_t padbuflen, uint8_t multiple)
77 {
78 	size_t	padlen;
79 
80 	padlen = multiple - (datalen % multiple);
81 	if (databuf == NULL)
82 		datalen = 0;
83 
84 	if (padlen > padbuflen - datalen) {
85 		return (CKR_DATA_LEN_RANGE);
86 	}
87 
88 	bcopy(databuf, padbuf, datalen);
89 	(void) memset(padbuf + datalen, padlen & 0xff, padlen);
90 
91 	return (0);
92 }
93 
94 /*
95  * 'padbuf' points to the recovered message.  Strip off the padding and
96  * validate it as much as possible.  'plen' is changed to hold the actual
97  * data length.  'padbuf' is unchanged.
98  */
99 int
100 pkcs7_decode(uint8_t *padbuf, size_t *plen)
101 {
102 	int	i;
103 	size_t	padlen;
104 
105 	/* Recover the padding value, even if padbuf has trailing nulls */
106 	while (*plen > 0 && (padlen = padbuf[*plen - 1]) == 0)
107 		(*plen)--;
108 
109 	/* Must have non-zero padding */
110 	if (padlen == 0)
111 		return (CKR_ENCRYPTED_DATA_INVALID);
112 
113 	/* Count back from all padding bytes; lint tag is for *plen-1-i >= 0 */
114 	/* LINTED E_SUSPICIOUS_COMPARISON */
115 	for (i = 0; i < padlen && (*plen - 1 - i) >= 0; i++) {
116 		if (padbuf[*plen - 1 - i] != (padlen & 0xff))
117 			return (CKR_ENCRYPTED_DATA_INVALID);
118 	}
119 	*plen -= i;
120 	return (0);
121 }
122