xref: /illumos-gate/usr/src/lib/pkcs11/pkcs11_softtoken/common/softEncrypt.c (revision 0ad0f0b2adb964c7bd56bbf5a831721e1a67beaf)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <pthread.h>
27 #include <security/cryptoki.h>
28 #include "softGlobal.h"
29 #include "softSession.h"
30 #include "softObject.h"
31 #include "softOps.h"
32 
33 
34 CK_RV
35 C_EncryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism,
36     CK_OBJECT_HANDLE hKey)
37 {
38 
39 	CK_RV		rv;
40 	soft_session_t	*session_p;
41 	soft_object_t	*key_p;
42 	boolean_t	lock_held = B_FALSE;
43 
44 	if (!softtoken_initialized)
45 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
46 
47 	/* Obtain the session pointer. */
48 	rv = handle2session(hSession, &session_p);
49 	if (rv != CKR_OK)
50 		return (rv);
51 
52 	if (pMechanism == NULL) {
53 		rv = CKR_ARGUMENTS_BAD;
54 		goto clean_exit;
55 	}
56 
57 	/* Obtain the object pointer. */
58 	HANDLE2OBJECT(hKey, key_p, rv);
59 	if (rv != CKR_OK)
60 		goto clean_exit;
61 
62 	/* Check to see if key object allows for encryption. */
63 	if (!(key_p->bool_attr_mask & ENCRYPT_BOOL_ON)) {
64 		rv = CKR_KEY_FUNCTION_NOT_PERMITTED;
65 		goto clean_exit1;
66 	}
67 
68 	(void) pthread_mutex_lock(&session_p->session_mutex);
69 	lock_held = B_TRUE;
70 
71 	/* Check to see if encrypt operation is already active. */
72 	if (session_p->encrypt.flags & CRYPTO_OPERATION_ACTIVE) {
73 		/* free the memory to avoid memory leak */
74 		soft_crypt_cleanup(session_p, B_TRUE, lock_held);
75 	}
76 
77 	/*
78 	 * This active flag will remain ON until application calls either
79 	 * C_Encrypt or C_EncryptFinal to actually obtain the final piece
80 	 * of ciphertext.
81 	 */
82 	session_p->encrypt.flags = CRYPTO_OPERATION_ACTIVE;
83 
84 	(void) pthread_mutex_unlock(&session_p->session_mutex);
85 	lock_held = B_FALSE;
86 
87 	rv = soft_encrypt_init(session_p, pMechanism, key_p);
88 
89 	if (rv != CKR_OK) {
90 		(void) pthread_mutex_lock(&session_p->session_mutex);
91 		session_p->encrypt.flags &= ~CRYPTO_OPERATION_ACTIVE;
92 		lock_held = B_TRUE;
93 	}
94 
95 clean_exit1:
96 	OBJ_REFRELE(key_p);
97 clean_exit:
98 	SES_REFRELE(session_p, lock_held);
99 	return (rv);
100 }
101 
102 
103 CK_RV
104 C_Encrypt(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pData, CK_ULONG ulDataLen,
105     CK_BYTE_PTR pEncryptedData, CK_ULONG_PTR pulEncryptedDataLen)
106 {
107 
108 	CK_RV		rv;
109 	soft_session_t	*session_p;
110 	boolean_t	lock_held = B_FALSE;
111 
112 	if (!softtoken_initialized)
113 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
114 
115 	/* Obtain the session pointer. */
116 	rv = handle2session(hSession, &session_p);
117 	if (rv != CKR_OK)
118 		return (rv);
119 
120 	/*
121 	 * Only check if input buffer is null.  How to handle zero input
122 	 * length depends on the mechanism in use.  For secret key mechanisms,
123 	 * unpadded ones yield zero length output, but padded ones always
124 	 * result in greater than zero length output.
125 	 */
126 	if (pData == NULL) {
127 		rv = CKR_ARGUMENTS_BAD;
128 		goto clean_exit;
129 	}
130 
131 	/*
132 	 * Only check if pulEncryptedDataLen is NULL.
133 	 * No need to check if pEncryptedData is NULL because
134 	 * application might just ask for the length of buffer to hold
135 	 * the ciphertext.
136 	 */
137 	if (pulEncryptedDataLen == NULL) {
138 		rv = CKR_ARGUMENTS_BAD;
139 		goto clean_exit;
140 	}
141 
142 	(void) pthread_mutex_lock(&session_p->session_mutex);
143 	lock_held = B_TRUE;
144 
145 	/* Application must call C_EncryptInit before calling C_Encrypt. */
146 	if (!(session_p->encrypt.flags & CRYPTO_OPERATION_ACTIVE)) {
147 		SES_REFRELE(session_p, lock_held);
148 		return (CKR_OPERATION_NOT_INITIALIZED);
149 	}
150 
151 	/*
152 	 * C_Encrypt must be called without intervening C_EncryptUpdate
153 	 * calls.
154 	 */
155 	if (session_p->encrypt.flags & CRYPTO_OPERATION_UPDATE) {
156 		/*
157 		 * C_Encrypt can not be used to terminate a multi-part
158 		 * operation, so we'll leave the active encrypt operation
159 		 * flag on and let the application continue with the
160 		 * encrypt update operation.
161 		 */
162 		SES_REFRELE(session_p, lock_held);
163 		return (CKR_FUNCTION_FAILED);
164 	}
165 
166 	(void) pthread_mutex_unlock(&session_p->session_mutex);
167 	lock_held = B_FALSE;
168 
169 	rv = soft_encrypt(session_p, pData, ulDataLen, pEncryptedData,
170 	    pulEncryptedDataLen);
171 
172 	if ((rv == CKR_BUFFER_TOO_SMALL) ||
173 	    (pEncryptedData == NULL && rv == CKR_OK)) {
174 		/*
175 		 * We will not terminate the active encrypt operation flag,
176 		 * when the application-supplied buffer is too small, or
177 		 * the application asks for the length of buffer to hold
178 		 * the ciphertext.
179 		 */
180 		SES_REFRELE(session_p, lock_held);
181 		return (rv);
182 	}
183 
184 clean_exit:
185 	/* Clear context, free key, and release session counter */
186 	soft_crypt_cleanup(session_p, B_TRUE, B_FALSE);
187 	return (rv);
188 }
189 
190 
191 CK_RV
192 C_EncryptUpdate(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pPart,
193     CK_ULONG ulPartLen, CK_BYTE_PTR pEncryptedPart,
194     CK_ULONG_PTR pulEncryptedPartLen)
195 {
196 
197 	CK_RV		rv;
198 	soft_session_t	*session_p;
199 	boolean_t	lock_held = B_FALSE;
200 
201 	if (!softtoken_initialized)
202 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
203 
204 	/* Obtain the session pointer. */
205 	rv = handle2session(hSession, &session_p);
206 	if (rv != CKR_OK)
207 		return (rv);
208 
209 	/*
210 	 * Only check if input buffer is null.  How to handle zero input
211 	 * length depends on the mechanism in use.  For secret key mechanisms,
212 	 * unpadded ones yeild zero length output, but padded ones always
213 	 * result in greater than zero length output.
214 	 */
215 	if (pPart == NULL) {
216 		rv = CKR_ARGUMENTS_BAD;
217 		goto clean_exit;
218 	}
219 
220 	/*
221 	 * Only check if pulEncryptedPartLen is NULL.
222 	 * No need to check if pEncryptedPart is NULL because
223 	 * application might just ask for the length of buffer to hold
224 	 * the ciphertext.
225 	 */
226 	if (pulEncryptedPartLen == NULL) {
227 		rv = CKR_ARGUMENTS_BAD;
228 		goto clean_exit;
229 	}
230 
231 	(void) pthread_mutex_lock(&session_p->session_mutex);
232 	lock_held = B_TRUE;
233 
234 	/*
235 	 * Application must call C_EncryptInit before calling
236 	 * C_EncryptUpdate.
237 	 */
238 	if (!(session_p->encrypt.flags & CRYPTO_OPERATION_ACTIVE)) {
239 		SES_REFRELE(session_p, lock_held);
240 		return (CKR_OPERATION_NOT_INITIALIZED);
241 	}
242 
243 	session_p->encrypt.flags |= CRYPTO_OPERATION_UPDATE;
244 
245 	(void) pthread_mutex_unlock(&session_p->session_mutex);
246 	lock_held = B_FALSE;
247 
248 	rv = soft_encrypt_update(session_p, pPart, ulPartLen,
249 	    pEncryptedPart, pulEncryptedPartLen);
250 
251 	/*
252 	 * If CKR_OK or CKR_BUFFER_TOO_SMALL, don't terminate the
253 	 * current encryption operation.
254 	 */
255 	if ((rv == CKR_OK) || (rv == CKR_BUFFER_TOO_SMALL)) {
256 		SES_REFRELE(session_p, lock_held);
257 		return (rv);
258 	}
259 
260 clean_exit:
261 	/*
262 	 * After an error occurred, terminate the current encrypt
263 	 * operation by resetting the active and update flags.
264 	 */
265 	soft_crypt_cleanup(session_p, B_TRUE, lock_held);
266 
267 	return (rv);
268 }
269 
270 
271 CK_RV
272 C_EncryptFinal(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pLastEncryptedPart,
273     CK_ULONG_PTR pulLastEncryptedPartLen)
274 {
275 
276 	CK_RV		rv;
277 	soft_session_t	*session_p;
278 	boolean_t	lock_held = B_FALSE;
279 
280 	if (!softtoken_initialized)
281 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
282 
283 	/* Obtain the session pointer. */
284 	rv = handle2session(hSession, &session_p);
285 	if (rv != CKR_OK)
286 		return (rv);
287 
288 	if (pulLastEncryptedPartLen == NULL) {
289 		rv = CKR_ARGUMENTS_BAD;
290 		goto clean_exit;
291 	}
292 
293 	(void) pthread_mutex_lock(&session_p->session_mutex);
294 	lock_held = B_TRUE;
295 
296 	/*
297 	 * Application must call C_EncryptInit before calling
298 	 * C_EncryptFinal.
299 	 */
300 	if (!(session_p->encrypt.flags & CRYPTO_OPERATION_ACTIVE)) {
301 		SES_REFRELE(session_p, lock_held);
302 		return (CKR_OPERATION_NOT_INITIALIZED);
303 	}
304 
305 	(void) pthread_mutex_unlock(&session_p->session_mutex);
306 	lock_held = B_FALSE;
307 
308 	rv = soft_encrypt_final(session_p, pLastEncryptedPart,
309 	    pulLastEncryptedPartLen);
310 
311 	if ((rv == CKR_BUFFER_TOO_SMALL) ||
312 	    (pLastEncryptedPart == NULL && rv == CKR_OK)) {
313 		/*
314 		 * We will not terminate the active encrypt operation flag,
315 		 * when the application-supplied buffer is too small, or
316 		 * the application asks for the length of buffer to hold
317 		 * the ciphertext.
318 		 */
319 		SES_REFRELE(session_p, lock_held);
320 		return (rv);
321 	}
322 
323 	/* Terminates the active encrypt operation. */
324 	(void) pthread_mutex_lock(&session_p->session_mutex);
325 	session_p->encrypt.flags = 0;
326 	lock_held = B_TRUE;
327 	SES_REFRELE(session_p, lock_held);
328 
329 	return (rv);
330 
331 clean_exit:
332 	/* Terminates the active encrypt operation. */
333 	soft_crypt_cleanup(session_p, B_TRUE, lock_held);
334 
335 	return (rv);
336 }
337