xref: /titanic_50/usr/src/lib/pkcs11/pkcs11_softtoken/common/softEncrypt.c (revision 06e1a7147edd272b7296f208141627a5b1191731)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <pthread.h>
29 #include <security/cryptoki.h>
30 #include "softGlobal.h"
31 #include "softSession.h"
32 #include "softObject.h"
33 #include "softOps.h"
34 
35 
36 CK_RV
37 C_EncryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism,
38     CK_OBJECT_HANDLE hKey)
39 {
40 
41 	CK_RV		rv;
42 	soft_session_t	*session_p;
43 	soft_object_t	*key_p;
44 	boolean_t	lock_held = B_FALSE;
45 
46 	if (!softtoken_initialized)
47 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
48 
49 	/* Obtain the session pointer. */
50 	rv = handle2session(hSession, &session_p);
51 	if (rv != CKR_OK)
52 		return (rv);
53 
54 	if (pMechanism == NULL) {
55 		rv = CKR_ARGUMENTS_BAD;
56 		goto clean_exit;
57 	}
58 
59 	/* Obtain the object pointer. */
60 	HANDLE2OBJECT(hKey, key_p, rv);
61 	if (rv != CKR_OK)
62 		goto clean_exit;
63 
64 	/* Check to see if key object allows for encryption. */
65 	if (!(key_p->bool_attr_mask & ENCRYPT_BOOL_ON)) {
66 		rv = CKR_KEY_TYPE_INCONSISTENT;
67 		goto clean_exit1;
68 	}
69 
70 	(void) pthread_mutex_lock(&session_p->session_mutex);
71 	lock_held = B_TRUE;
72 
73 	/* Check to see if encrypt operation is already active. */
74 	if (session_p->encrypt.flags & CRYPTO_OPERATION_ACTIVE) {
75 		/* free the memory to avoid memory leak */
76 		soft_crypt_cleanup(session_p, B_TRUE, lock_held);
77 	}
78 
79 	/*
80 	 * This active flag will remain ON until application calls either
81 	 * C_Encrypt or C_EncryptFinal to actually obtain the final piece
82 	 * of ciphertext.
83 	 */
84 	session_p->encrypt.flags = CRYPTO_OPERATION_ACTIVE;
85 
86 	(void) pthread_mutex_unlock(&session_p->session_mutex);
87 	lock_held = B_FALSE;
88 
89 	rv = soft_encrypt_init(session_p, pMechanism, key_p);
90 
91 	if (rv != CKR_OK) {
92 		(void) pthread_mutex_lock(&session_p->session_mutex);
93 		session_p->encrypt.flags &= ~CRYPTO_OPERATION_ACTIVE;
94 		lock_held = B_TRUE;
95 	}
96 
97 clean_exit1:
98 	OBJ_REFRELE(key_p);
99 clean_exit:
100 	SES_REFRELE(session_p, lock_held);
101 	return (rv);
102 }
103 
104 
105 CK_RV
106 C_Encrypt(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pData, CK_ULONG ulDataLen,
107     CK_BYTE_PTR pEncryptedData, CK_ULONG_PTR pulEncryptedDataLen)
108 {
109 
110 	CK_RV		rv;
111 	soft_session_t	*session_p;
112 	boolean_t	lock_held = B_FALSE;
113 
114 	if (!softtoken_initialized)
115 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
116 
117 	/* Obtain the session pointer. */
118 	rv = handle2session(hSession, &session_p);
119 	if (rv != CKR_OK)
120 		return (rv);
121 
122 	/*
123 	 * Only check if input buffer is null.  How to handle zero input
124 	 * length depends on the mechanism in use.  For secret key mechanisms,
125 	 * unpadded ones yield zero length output, but padded ones always
126 	 * result in greater than zero length output.
127 	 */
128 	if (pData == NULL) {
129 		rv = CKR_ARGUMENTS_BAD;
130 		goto clean_exit;
131 	}
132 
133 	/*
134 	 * Only check if pulEncryptedDataLen is NULL.
135 	 * No need to check if pEncryptedData is NULL because
136 	 * application might just ask for the length of buffer to hold
137 	 * the ciphertext.
138 	 */
139 	if (pulEncryptedDataLen == NULL) {
140 		rv = CKR_ARGUMENTS_BAD;
141 		goto clean_exit;
142 	}
143 
144 	(void) pthread_mutex_lock(&session_p->session_mutex);
145 	lock_held = B_TRUE;
146 
147 	/* Application must call C_EncryptInit before calling C_Encrypt. */
148 	if (!(session_p->encrypt.flags & CRYPTO_OPERATION_ACTIVE)) {
149 		SES_REFRELE(session_p, lock_held);
150 		return (CKR_OPERATION_NOT_INITIALIZED);
151 	}
152 
153 	/*
154 	 * C_Encrypt must be called without intervening C_EncryptUpdate
155 	 * calls.
156 	 */
157 	if (session_p->encrypt.flags & CRYPTO_OPERATION_UPDATE) {
158 		/*
159 		 * C_Encrypt can not be used to terminate a multi-part
160 		 * operation, so we'll leave the active encrypt operation
161 		 * flag on and let the application continue with the
162 		 * encrypt update operation.
163 		 */
164 		SES_REFRELE(session_p, lock_held);
165 		return (CKR_FUNCTION_FAILED);
166 	}
167 
168 	(void) pthread_mutex_unlock(&session_p->session_mutex);
169 	lock_held = B_FALSE;
170 
171 	rv = soft_encrypt(session_p, pData, ulDataLen, pEncryptedData,
172 	    pulEncryptedDataLen);
173 
174 	if ((rv == CKR_BUFFER_TOO_SMALL) ||
175 	    (pEncryptedData == NULL && rv == CKR_OK)) {
176 		/*
177 		 * We will not terminate the active encrypt operation flag,
178 		 * when the application-supplied buffer is too small, or
179 		 * the application asks for the length of buffer to hold
180 		 * the ciphertext.
181 		 */
182 		SES_REFRELE(session_p, lock_held);
183 		return (rv);
184 	}
185 
186 	/*
187 	 * Normal exit.
188 	 * Terminates the active encrypt operation.
189 	 * Application needs to call C_EncryptInit again for next
190 	 * encrypt operation.
191 	 */
192 	(void) pthread_mutex_lock(&session_p->session_mutex);
193 	session_p->encrypt.flags = 0;
194 	lock_held = B_TRUE;
195 	SES_REFRELE(session_p, lock_held);
196 	return (rv);
197 
198 clean_exit:
199 	soft_crypt_cleanup(session_p, B_TRUE, lock_held);
200 
201 	return (rv);
202 }
203 
204 
205 CK_RV
206 C_EncryptUpdate(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pPart,
207     CK_ULONG ulPartLen, CK_BYTE_PTR pEncryptedPart,
208     CK_ULONG_PTR pulEncryptedPartLen)
209 {
210 
211 	CK_RV		rv;
212 	soft_session_t	*session_p;
213 	boolean_t	lock_held = B_FALSE;
214 
215 	if (!softtoken_initialized)
216 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
217 
218 	/* Obtain the session pointer. */
219 	rv = handle2session(hSession, &session_p);
220 	if (rv != CKR_OK)
221 		return (rv);
222 
223 	/*
224 	 * Only check if input buffer is null.  How to handle zero input
225 	 * length depends on the mechanism in use.  For secret key mechanisms,
226 	 * unpadded ones yeild zero length output, but padded ones always
227 	 * result in greater than zero length output.
228 	 */
229 	if (pPart == NULL) {
230 		rv = CKR_ARGUMENTS_BAD;
231 		goto clean_exit;
232 	}
233 
234 	/*
235 	 * Only check if pulEncryptedPartLen is NULL.
236 	 * No need to check if pEncryptedPart is NULL because
237 	 * application might just ask for the length of buffer to hold
238 	 * the ciphertext.
239 	 */
240 	if (pulEncryptedPartLen == NULL) {
241 		rv = CKR_ARGUMENTS_BAD;
242 		goto clean_exit;
243 	}
244 
245 	(void) pthread_mutex_lock(&session_p->session_mutex);
246 	lock_held = B_TRUE;
247 
248 	/*
249 	 * Application must call C_EncryptInit before calling
250 	 * C_EncryptUpdate.
251 	 */
252 	if (!(session_p->encrypt.flags & CRYPTO_OPERATION_ACTIVE)) {
253 		SES_REFRELE(session_p, lock_held);
254 		return (CKR_OPERATION_NOT_INITIALIZED);
255 	}
256 
257 	session_p->encrypt.flags |= CRYPTO_OPERATION_UPDATE;
258 
259 	(void) pthread_mutex_unlock(&session_p->session_mutex);
260 	lock_held = B_FALSE;
261 
262 	rv = soft_encrypt_update(session_p, pPart, ulPartLen,
263 	    pEncryptedPart, pulEncryptedPartLen);
264 
265 	/*
266 	 * If CKR_OK or CKR_BUFFER_TOO_SMALL, don't terminate the
267 	 * current encryption operation.
268 	 */
269 	if ((rv == CKR_OK) || (rv == CKR_BUFFER_TOO_SMALL)) {
270 		SES_REFRELE(session_p, lock_held);
271 		return (rv);
272 	}
273 
274 clean_exit:
275 	/*
276 	 * After an error occurred, terminate the current encrypt
277 	 * operation by resetting the active and update flags.
278 	 */
279 	soft_crypt_cleanup(session_p, B_TRUE, lock_held);
280 
281 	return (rv);
282 }
283 
284 
285 CK_RV
286 C_EncryptFinal(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pLastEncryptedPart,
287     CK_ULONG_PTR pulLastEncryptedPartLen)
288 {
289 
290 	CK_RV		rv;
291 	soft_session_t	*session_p;
292 	boolean_t	lock_held = B_FALSE;
293 
294 	if (!softtoken_initialized)
295 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
296 
297 	/* Obtain the session pointer. */
298 	rv = handle2session(hSession, &session_p);
299 	if (rv != CKR_OK)
300 		return (rv);
301 
302 	if (pulLastEncryptedPartLen == NULL) {
303 		rv = CKR_ARGUMENTS_BAD;
304 		goto clean_exit;
305 	}
306 
307 	(void) pthread_mutex_lock(&session_p->session_mutex);
308 	lock_held = B_TRUE;
309 
310 	/*
311 	 * Application must call C_EncryptInit before calling
312 	 * C_EncryptFinal.
313 	 */
314 	if (!(session_p->encrypt.flags & CRYPTO_OPERATION_ACTIVE)) {
315 		SES_REFRELE(session_p, lock_held);
316 		return (CKR_OPERATION_NOT_INITIALIZED);
317 	}
318 
319 	(void) pthread_mutex_unlock(&session_p->session_mutex);
320 	lock_held = B_FALSE;
321 
322 	rv = soft_encrypt_final(session_p, pLastEncryptedPart,
323 	    pulLastEncryptedPartLen);
324 
325 	if ((rv == CKR_BUFFER_TOO_SMALL) ||
326 	    (pLastEncryptedPart == NULL && rv == CKR_OK)) {
327 		/*
328 		 * We will not terminate the active encrypt operation flag,
329 		 * when the application-supplied buffer is too small, or
330 		 * the application asks for the length of buffer to hold
331 		 * the ciphertext.
332 		 */
333 		SES_REFRELE(session_p, lock_held);
334 		return (rv);
335 	}
336 
337 	/* Terminates the active encrypt operation. */
338 	(void) pthread_mutex_lock(&session_p->session_mutex);
339 	session_p->encrypt.flags = 0;
340 	lock_held = B_TRUE;
341 	SES_REFRELE(session_p, lock_held);
342 
343 	return (rv);
344 
345 clean_exit:
346 	/* Terminates the active encrypt operation. */
347 	soft_crypt_cleanup(session_p, B_TRUE, lock_held);
348 
349 	return (rv);
350 }
351