xref: /titanic_50/usr/src/lib/pkcs11/pkcs11_softtoken/common/softEncrypt.c (revision b1dd958f54f8bfa984d306bb8ca8264855761d7b)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 #include <pthread.h>
30 #include <security/cryptoki.h>
31 #include "softGlobal.h"
32 #include "softSession.h"
33 #include "softObject.h"
34 #include "softOps.h"
35 
36 
37 CK_RV
38 C_EncryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism,
39     CK_OBJECT_HANDLE hKey)
40 {
41 
42 	CK_RV		rv;
43 	soft_session_t	*session_p;
44 	soft_object_t	*key_p;
45 	boolean_t	lock_held = B_FALSE;
46 
47 	if (!softtoken_initialized)
48 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
49 
50 	/* Obtain the session pointer. */
51 	rv = handle2session(hSession, &session_p);
52 	if (rv != CKR_OK)
53 		return (rv);
54 
55 	if (pMechanism == NULL) {
56 		rv = CKR_ARGUMENTS_BAD;
57 		goto clean_exit;
58 	}
59 
60 	/* Obtain the object pointer. */
61 	HANDLE2OBJECT(hKey, key_p, rv);
62 	if (rv != CKR_OK)
63 		goto clean_exit;
64 
65 	/* Check to see if key object allows for encryption. */
66 	if (!(key_p->bool_attr_mask & ENCRYPT_BOOL_ON)) {
67 		rv = CKR_KEY_TYPE_INCONSISTENT;
68 		goto clean_exit1;
69 	}
70 
71 	(void) pthread_mutex_lock(&session_p->session_mutex);
72 	lock_held = B_TRUE;
73 
74 	/* Check to see if encrypt operation is already active. */
75 	if (session_p->encrypt.flags & CRYPTO_OPERATION_ACTIVE) {
76 		/* free the memory to avoid memory leak */
77 		soft_crypt_cleanup(session_p, B_TRUE, lock_held);
78 	}
79 
80 	/*
81 	 * This active flag will remain ON until application calls either
82 	 * C_Encrypt or C_EncryptFinal to actually obtain the final piece
83 	 * of ciphertext.
84 	 */
85 	session_p->encrypt.flags = CRYPTO_OPERATION_ACTIVE;
86 
87 	(void) pthread_mutex_unlock(&session_p->session_mutex);
88 	lock_held = B_FALSE;
89 
90 	rv = soft_encrypt_init(session_p, pMechanism, key_p);
91 
92 	if (rv != CKR_OK) {
93 		(void) pthread_mutex_lock(&session_p->session_mutex);
94 		session_p->encrypt.flags &= ~CRYPTO_OPERATION_ACTIVE;
95 		lock_held = B_TRUE;
96 	}
97 
98 clean_exit1:
99 	OBJ_REFRELE(key_p);
100 clean_exit:
101 	SES_REFRELE(session_p, lock_held);
102 	return (rv);
103 }
104 
105 
106 CK_RV
107 C_Encrypt(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pData, CK_ULONG ulDataLen,
108     CK_BYTE_PTR pEncryptedData, CK_ULONG_PTR pulEncryptedDataLen)
109 {
110 
111 	CK_RV		rv;
112 	soft_session_t	*session_p;
113 	boolean_t	lock_held = B_FALSE;
114 
115 	if (!softtoken_initialized)
116 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
117 
118 	/* Obtain the session pointer. */
119 	rv = handle2session(hSession, &session_p);
120 	if (rv != CKR_OK)
121 		return (rv);
122 
123 	if (ulDataLen == 0) {
124 		SES_REFRELE(session_p, lock_held);
125 		return (CKR_OK);
126 	}
127 
128 	if (pData == NULL) {
129 		rv = CKR_ARGUMENTS_BAD;
130 		goto clean_exit;
131 	}
132 
133 	/*
134 	 * Only check if pulEncryptedDataLen is NULL.
135 	 * No need to check if pEncryptedData is NULL because
136 	 * application might just ask for the length of buffer to hold
137 	 * the ciphertext.
138 	 */
139 	if (pulEncryptedDataLen == NULL) {
140 		rv = CKR_ARGUMENTS_BAD;
141 		goto clean_exit;
142 	}
143 
144 	(void) pthread_mutex_lock(&session_p->session_mutex);
145 	lock_held = B_TRUE;
146 
147 	/* Application must call C_EncryptInit before calling C_Encrypt. */
148 	if (!(session_p->encrypt.flags & CRYPTO_OPERATION_ACTIVE)) {
149 		SES_REFRELE(session_p, lock_held);
150 		return (CKR_OPERATION_NOT_INITIALIZED);
151 	}
152 
153 	/*
154 	 * C_Encrypt must be called without intervening C_EncryptUpdate
155 	 * calls.
156 	 */
157 	if (session_p->encrypt.flags & CRYPTO_OPERATION_UPDATE) {
158 		/*
159 		 * C_Encrypt can not be used to terminate a multi-part
160 		 * operation, so we'll leave the active encrypt operation
161 		 * flag on and let the application continue with the
162 		 * encrypt update operation.
163 		 */
164 		SES_REFRELE(session_p, lock_held);
165 		return (CKR_FUNCTION_FAILED);
166 	}
167 
168 	(void) pthread_mutex_unlock(&session_p->session_mutex);
169 	lock_held = B_FALSE;
170 
171 	rv = soft_encrypt(session_p, pData, ulDataLen, pEncryptedData,
172 	    pulEncryptedDataLen);
173 
174 	if ((rv == CKR_BUFFER_TOO_SMALL) ||
175 	    (pEncryptedData == NULL && rv == CKR_OK)) {
176 		/*
177 		 * We will not terminate the active encrypt operation flag,
178 		 * when the application-supplied buffer is too small, or
179 		 * the application asks for the length of buffer to hold
180 		 * the ciphertext.
181 		 */
182 		SES_REFRELE(session_p, lock_held);
183 		return (rv);
184 	}
185 
186 	/*
187 	 * Normal exit.
188 	 * Terminates the active encrypt operation.
189 	 * Application needs to call C_EncryptInit again for next
190 	 * encrypt operation.
191 	 */
192 	(void) pthread_mutex_lock(&session_p->session_mutex);
193 	session_p->encrypt.flags = 0;
194 	lock_held = B_TRUE;
195 	SES_REFRELE(session_p, lock_held);
196 	return (rv);
197 
198 clean_exit:
199 	soft_crypt_cleanup(session_p, B_TRUE, lock_held);
200 
201 	return (rv);
202 }
203 
204 
205 CK_RV
206 C_EncryptUpdate(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pPart,
207     CK_ULONG ulPartLen, CK_BYTE_PTR pEncryptedPart,
208     CK_ULONG_PTR pulEncryptedPartLen)
209 {
210 
211 	CK_RV		rv;
212 	soft_session_t	*session_p;
213 	boolean_t	lock_held = B_FALSE;
214 
215 	if (!softtoken_initialized)
216 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
217 
218 	/* Obtain the session pointer. */
219 	rv = handle2session(hSession, &session_p);
220 	if (rv != CKR_OK)
221 		return (rv);
222 
223 	if (ulPartLen == 0) {
224 		SES_REFRELE(session_p, lock_held);
225 		return (CKR_OK);
226 	}
227 
228 	if (pPart == NULL) {
229 		rv = CKR_ARGUMENTS_BAD;
230 		goto clean_exit;
231 	}
232 
233 	/*
234 	 * Only check if pulEncryptedPartLen is NULL.
235 	 * No need to check if pEncryptedPart is NULL because
236 	 * application might just ask for the length of buffer to hold
237 	 * the ciphertext.
238 	 */
239 	if (pulEncryptedPartLen == NULL) {
240 		rv = CKR_ARGUMENTS_BAD;
241 		goto clean_exit;
242 	}
243 
244 	(void) pthread_mutex_lock(&session_p->session_mutex);
245 	lock_held = B_TRUE;
246 
247 	/*
248 	 * Application must call C_EncryptInit before calling
249 	 * C_EncryptUpdate.
250 	 */
251 	if (!(session_p->encrypt.flags & CRYPTO_OPERATION_ACTIVE)) {
252 		SES_REFRELE(session_p, lock_held);
253 		return (CKR_OPERATION_NOT_INITIALIZED);
254 	}
255 
256 	session_p->encrypt.flags |= CRYPTO_OPERATION_UPDATE;
257 
258 	(void) pthread_mutex_unlock(&session_p->session_mutex);
259 	lock_held = B_FALSE;
260 
261 	rv = soft_encrypt_update(session_p, pPart, ulPartLen,
262 	    pEncryptedPart, pulEncryptedPartLen);
263 
264 	/*
265 	 * If CKR_OK or CKR_BUFFER_TOO_SMALL, don't terminate the
266 	 * current encryption operation.
267 	 */
268 	if ((rv == CKR_OK) || (rv == CKR_BUFFER_TOO_SMALL)) {
269 		SES_REFRELE(session_p, lock_held);
270 		return (rv);
271 	}
272 
273 clean_exit:
274 	/*
275 	 * After an error occurred, terminate the current encrypt
276 	 * operation by resetting the active and update flags.
277 	 */
278 	soft_crypt_cleanup(session_p, B_TRUE, lock_held);
279 
280 	return (rv);
281 }
282 
283 
284 CK_RV
285 C_EncryptFinal(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pLastEncryptedPart,
286     CK_ULONG_PTR pulLastEncryptedPartLen)
287 {
288 
289 	CK_RV		rv;
290 	soft_session_t	*session_p;
291 	boolean_t	lock_held = B_FALSE;
292 
293 	if (!softtoken_initialized)
294 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
295 
296 	/* Obtain the session pointer. */
297 	rv = handle2session(hSession, &session_p);
298 	if (rv != CKR_OK)
299 		return (rv);
300 
301 	if (pulLastEncryptedPartLen == NULL) {
302 		rv = CKR_ARGUMENTS_BAD;
303 		goto clean_exit;
304 	}
305 
306 	(void) pthread_mutex_lock(&session_p->session_mutex);
307 	lock_held = B_TRUE;
308 
309 	/*
310 	 * Application must call C_EncryptInit before calling
311 	 * C_EncryptFinal.
312 	 */
313 	if (!(session_p->encrypt.flags & CRYPTO_OPERATION_ACTIVE)) {
314 		SES_REFRELE(session_p, lock_held);
315 		return (CKR_OPERATION_NOT_INITIALIZED);
316 	}
317 
318 	(void) pthread_mutex_unlock(&session_p->session_mutex);
319 	lock_held = B_FALSE;
320 
321 	rv = soft_encrypt_final(session_p, pLastEncryptedPart,
322 	    pulLastEncryptedPartLen);
323 
324 	if ((rv == CKR_BUFFER_TOO_SMALL) ||
325 	    (pLastEncryptedPart == NULL && rv == CKR_OK)) {
326 		/*
327 		 * We will not terminate the active encrypt operation flag,
328 		 * when the application-supplied buffer is too small, or
329 		 * the application asks for the length of buffer to hold
330 		 * the ciphertext.
331 		 */
332 		SES_REFRELE(session_p, lock_held);
333 		return (rv);
334 	}
335 
336 	/* Terminates the active encrypt operation. */
337 	(void) pthread_mutex_lock(&session_p->session_mutex);
338 	session_p->encrypt.flags = 0;
339 	lock_held = B_TRUE;
340 	SES_REFRELE(session_p, lock_held);
341 
342 	return (rv);
343 
344 clean_exit:
345 	/* Terminates the active encrypt operation. */
346 	soft_crypt_cleanup(session_p, B_TRUE, lock_held);
347 
348 	return (rv);
349 }
350