xref: /illumos-gate/usr/src/lib/pkcs11/pkcs11_softtoken/common/softDigest.c (revision 8475e04352e630e4bd0f59a283286ee2475a14ce)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <pthread.h>
27 #include <security/cryptoki.h>
28 #include "softGlobal.h"
29 #include "softOps.h"
30 #include "softSession.h"
31 
32 
33 CK_RV
34 C_DigestInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism)
35 {
36 
37 	CK_RV		rv;
38 	soft_session_t	*session_p;
39 	boolean_t lock_held = B_TRUE;
40 
41 	if (!softtoken_initialized)
42 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
43 
44 	/*
45 	 * Obtain the session pointer. Also, increment the session
46 	 * reference count.
47 	 */
48 	rv = handle2session(hSession, &session_p);
49 	if (rv != CKR_OK)
50 		return (rv);
51 
52 	if (pMechanism == NULL) {
53 		rv = CKR_ARGUMENTS_BAD;
54 		goto clean_exit;
55 	}
56 
57 	/* Acquire the session lock */
58 	(void) pthread_mutex_lock(&session_p->session_mutex);
59 
60 	/* Check to see if digest operation is already active */
61 	if (session_p->digest.flags & CRYPTO_OPERATION_ACTIVE) {
62 		/*
63 		 * Free the memory to avoid memory leak.
64 		 * digest.context is only a flat structure.
65 		 */
66 		soft_digest_cleanup(session_p, lock_held);
67 	}
68 
69 	/*
70 	 * This active flag will remain ON until application calls either
71 	 * C_Digest or C_DigestFinal to actually obtain the value of
72 	 * the message digest.
73 	 */
74 	session_p->digest.flags = CRYPTO_OPERATION_ACTIVE;
75 
76 	(void) pthread_mutex_unlock(&session_p->session_mutex);
77 
78 	rv = soft_digest_init(session_p, pMechanism);
79 
80 	if (rv != CKR_OK) {
81 		(void) pthread_mutex_lock(&session_p->session_mutex);
82 		session_p->digest.flags &= ~CRYPTO_OPERATION_ACTIVE;
83 		/*
84 		 * Decrement the session reference count.
85 		 * We hold the session lock, and SES_REFRELE()
86 		 * will release the session lock for us.
87 		 */
88 		SES_REFRELE(session_p, lock_held);
89 		return (rv);
90 	}
91 
92 clean_exit:
93 	/*
94 	 * Decrement the session reference count.
95 	 * We do not hold the session lock.
96 	 */
97 	lock_held = B_FALSE;
98 	SES_REFRELE(session_p, lock_held);
99 	return (rv);
100 }
101 
102 
103 CK_RV
104 C_Digest(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pData, CK_ULONG ulDataLen,
105     CK_BYTE_PTR pDigest, CK_ULONG_PTR pulDigestLen)
106 {
107 
108 	CK_RV		rv;
109 	soft_session_t	*session_p;
110 	boolean_t lock_held = B_TRUE;
111 
112 	if (!softtoken_initialized)
113 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
114 
115 	/*
116 	 * Obtain the session pointer. Also, increment the session
117 	 * reference count.
118 	 */
119 	rv = handle2session(hSession, &session_p);
120 	if (rv != CKR_OK)
121 		return (rv);
122 
123 	if (pData == NULL || pulDigestLen == NULL) {
124 		rv = CKR_ARGUMENTS_BAD;
125 		goto clean_exit;
126 	}
127 
128 	/* Acquire the session lock */
129 	(void) pthread_mutex_lock(&session_p->session_mutex);
130 
131 	/* Application must call C_DigestInit before calling C_Digest */
132 	if (!(session_p->digest.flags & CRYPTO_OPERATION_ACTIVE)) {
133 		/*
134 		 * Decrement the session reference count.
135 		 * We hold the session lock, and SES_REFRELE()
136 		 * will release the session lock for us.
137 		 */
138 		SES_REFRELE(session_p, lock_held);
139 		return (CKR_OPERATION_NOT_INITIALIZED);
140 	}
141 
142 	/*
143 	 * C_Digest must be called without intervening C_DigestUpdate
144 	 * calls.
145 	 */
146 	if (session_p->digest.flags & CRYPTO_OPERATION_UPDATE) {
147 		/*
148 		 * C_Digest can not be used to terminate a multi-part
149 		 * operation, so we'll leave the active digest operation
150 		 * flag on and let the application continue with the
151 		 * digest update operation.
152 		 *
153 		 * Decrement the session reference count.
154 		 * We hold the session lock, and SES_REFRELE()
155 		 * will release the session lock for us.
156 		 */
157 		SES_REFRELE(session_p, lock_held);
158 		return (CKR_FUNCTION_FAILED);
159 	}
160 
161 	(void) pthread_mutex_unlock(&session_p->session_mutex);
162 
163 	rv = soft_digest(session_p, pData, ulDataLen, pDigest, pulDigestLen);
164 
165 	if ((rv == CKR_BUFFER_TOO_SMALL) ||
166 	    (pDigest == NULL && rv == CKR_OK)) {
167 		/*
168 		 * We will not terminate the active digest operation flag,
169 		 * when the application-supplied buffer is too small, or
170 		 * the application asks for the length of buffer to hold
171 		 * the message digest.
172 		 *
173 		 * Decrement the session reference count.
174 		 * We do not hold the session lock.
175 		 */
176 		lock_held = B_FALSE;
177 		SES_REFRELE(session_p, lock_held);
178 		return (rv);
179 	}
180 
181 clean_exit:
182 	/*
183 	 * Terminates the active digest operation.
184 	 * Application needs to call C_DigestInit again for next
185 	 * digest operation.
186 	 */
187 	(void) pthread_mutex_lock(&session_p->session_mutex);
188 
189 	soft_digest_cleanup(session_p, lock_held);
190 
191 	/*
192 	 * Decrement the session reference count.
193 	 * We hold the session lock, and SES_REFRELE()
194 	 * will release the session lock for us.
195 	 */
196 	SES_REFRELE(session_p, lock_held);
197 
198 	return (rv);
199 }
200 
201 
202 CK_RV
203 C_DigestUpdate(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pPart,
204     CK_ULONG ulPartLen)
205 {
206 
207 	CK_RV		rv;
208 	soft_session_t	*session_p;
209 	boolean_t lock_held = B_TRUE;
210 
211 	if (!softtoken_initialized)
212 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
213 
214 	/*
215 	 * Obtain the session pointer. Also, increment the session
216 	 * reference count.
217 	 */
218 	rv = handle2session(hSession, &session_p);
219 	if (rv != CKR_OK)
220 		return (rv);
221 
222 	if (pPart == NULL) {
223 		rv = CKR_ARGUMENTS_BAD;
224 		goto clean_exit;
225 	}
226 
227 	/* Acquire the session lock */
228 	(void) pthread_mutex_lock(&session_p->session_mutex);
229 
230 	/*
231 	 * Application must call C_DigestInit before calling
232 	 * C_DigestUpdate.
233 	 */
234 	if (!(session_p->digest.flags & CRYPTO_OPERATION_ACTIVE)) {
235 		/*
236 		 * Decrement the session reference count.
237 		 * We hold the session lock, and SES_REFRELE()
238 		 * will release the session lock for us.
239 		 */
240 		SES_REFRELE(session_p, lock_held);
241 		return (CKR_OPERATION_NOT_INITIALIZED);
242 	}
243 
244 	/* Set update flag to protect C_Digest */
245 	session_p->digest.flags |= CRYPTO_OPERATION_UPDATE;
246 
247 	(void) pthread_mutex_unlock(&session_p->session_mutex);
248 
249 	rv = soft_digest_update(session_p, pPart, ulPartLen);
250 
251 	if (rv == CKR_OK) {
252 		/*
253 		 * Decrement the session reference count.
254 		 * We do not hold the session lock.
255 		 */
256 		lock_held = B_FALSE;
257 		SES_REFRELE(session_p, lock_held);
258 		return (CKR_OK);
259 	}
260 
261 clean_exit:
262 	/*
263 	 * After an error occurred, terminate the current digest
264 	 * operation by resetting the active and update flags.
265 	 */
266 	(void) pthread_mutex_lock(&session_p->session_mutex);
267 
268 	soft_digest_cleanup(session_p, lock_held);
269 
270 	/*
271 	 * Decrement the session reference count.
272 	 * We hold the session lock, and SES_REFRELE()
273 	 * will release the session lock for us.
274 	 */
275 	SES_REFRELE(session_p, lock_held);
276 
277 	return (rv);
278 }
279 
280 
281 CK_RV
282 C_DigestKey(CK_SESSION_HANDLE hSession, CK_OBJECT_HANDLE hKey)
283 {
284 
285 	CK_RV		rv;
286 	soft_session_t	*session_p;
287 	soft_object_t	*key_p;
288 	boolean_t lock_held = B_TRUE;
289 
290 	if (!softtoken_initialized)
291 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
292 
293 	/*
294 	 * Obtain the session pointer. Also, increment the session
295 	 * reference count.
296 	 */
297 	rv = handle2session(hSession, &session_p);
298 	if (rv != CKR_OK)
299 		return (rv);
300 
301 	/* Obtain the object pointer. */
302 	HANDLE2OBJECT(hKey, key_p, rv);
303 	if (rv != CKR_OK)
304 		goto clean_exit;
305 
306 	(void) pthread_mutex_lock(&session_p->session_mutex);
307 
308 	/*
309 	 * Application must call C_DigestInit before calling
310 	 * C_DigestKey.
311 	 */
312 	if (!(session_p->digest.flags & CRYPTO_OPERATION_ACTIVE)) {
313 		/*
314 		 * Decrement the session reference count.
315 		 * We hold the session lock, and SES_REFRELE()
316 		 * will release the session lock for us.
317 		 */
318 		OBJ_REFRELE(key_p);
319 		SES_REFRELE(session_p, lock_held);
320 		return (CKR_OPERATION_NOT_INITIALIZED);
321 	}
322 
323 	/*
324 	 * Remember the fact that a key was thrown into the mix, so that
325 	 * C_DigestFinal bzero()'s the digest context before freeing it.
326 	 */
327 	session_p->digest.flags |= (CRYPTO_KEY_DIGESTED |
328 	    CRYPTO_OPERATION_UPDATE);
329 
330 	(void) pthread_mutex_unlock(&session_p->session_mutex);
331 
332 	rv = soft_digest_key(session_p, key_p);
333 
334 	if (rv == CKR_OK) {
335 		/*
336 		 * Decrement the session reference count.
337 		 * We do not hold the session lock.
338 		 */
339 		lock_held = B_FALSE;
340 		OBJ_REFRELE(key_p);
341 		SES_REFRELE(session_p, lock_held);
342 		return (CKR_OK);
343 	}
344 
345 	OBJ_REFRELE(key_p);
346 clean_exit:
347 	/*
348 	 * After an error occurred, terminate the current digest
349 	 * operation by resetting the active and update flags.
350 	 */
351 	(void) pthread_mutex_lock(&session_p->session_mutex);
352 
353 	soft_digest_cleanup(session_p, lock_held);
354 
355 	/*
356 	 * Decrement the session reference count.
357 	 * We hold the session lock, and SES_REFRELE()
358 	 * will release the session lock for us.
359 	 */
360 	SES_REFRELE(session_p, lock_held);
361 	return (rv);
362 }
363 
364 
365 CK_RV
366 C_DigestFinal(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pDigest,
367     CK_ULONG_PTR pulDigestLen)
368 {
369 
370 	CK_RV		rv;
371 	soft_session_t	*session_p;
372 	boolean_t lock_held = B_TRUE;
373 
374 	if (!softtoken_initialized)
375 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
376 
377 	/*
378 	 * Obtain the session pointer. Also, increment the session
379 	 * reference count.
380 	 */
381 	rv = handle2session(hSession, &session_p);
382 	if (rv != CKR_OK)
383 		return (rv);
384 
385 	if (pulDigestLen == NULL) {
386 		rv = CKR_ARGUMENTS_BAD;
387 		goto clean_exit;
388 	}
389 
390 	/* Acquire the session lock */
391 	(void) pthread_mutex_lock(&session_p->session_mutex);
392 
393 	/*
394 	 * Application must call C_DigestInit before calling
395 	 * C_DigestFinal.
396 	 */
397 	if (!(session_p->digest.flags & CRYPTO_OPERATION_ACTIVE)) {
398 		/*
399 		 * Decrement the session reference count.
400 		 * We hold the session lock, and SES_REFRELE()
401 		 * will release the session lock for us.
402 		 */
403 		SES_REFRELE(session_p, lock_held);
404 		return (CKR_OPERATION_NOT_INITIALIZED);
405 	}
406 
407 	(void) pthread_mutex_unlock(&session_p->session_mutex);
408 
409 	rv = soft_digest_final(session_p, pDigest, pulDigestLen);
410 
411 	if ((rv == CKR_BUFFER_TOO_SMALL) ||
412 	    (pDigest == NULL && rv == CKR_OK)) {
413 		/*
414 		 * We will not terminate the active digest operation flag,
415 		 * when the application-supplied buffer is too small, or
416 		 * the application asks for the length of buffer to hold
417 		 * the message digest.
418 		 *
419 		 * Decrement the session reference count.
420 		 * We do not hold the session lock.
421 		 */
422 		lock_held = B_FALSE;
423 		SES_REFRELE(session_p, lock_held);
424 		return (rv);
425 	}
426 
427 clean_exit:
428 	/* Terminates the active digest operation */
429 	(void) pthread_mutex_lock(&session_p->session_mutex);
430 
431 	soft_digest_cleanup(session_p, lock_held);
432 
433 	/*
434 	 * Decrement the session reference count.
435 	 * We hold the session lock, and SES_REFRELE()
436 	 * will release the session lock for us.
437 	 */
438 	SES_REFRELE(session_p, lock_held);
439 
440 	return (rv);
441 }
442