xref: /illumos-gate/usr/src/lib/pkcs11/pkcs11_softtoken/common/softDigest.c (revision 66582b606a8194f7f3ba5b3a3a6dca5b0d346361)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  * Copyright 2018, Joyent, Inc.
25  */
26 
27 #include <pthread.h>
28 #include <security/cryptoki.h>
29 #include "softGlobal.h"
30 #include "softOps.h"
31 #include "softSession.h"
32 
33 
34 CK_RV
35 C_DigestInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism)
36 {
37 
38 	CK_RV		rv;
39 	soft_session_t	*session_p;
40 	boolean_t lock_held = B_TRUE;
41 
42 	if (!softtoken_initialized)
43 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
44 
45 	/*
46 	 * Obtain the session pointer. Also, increment the session
47 	 * reference count.
48 	 */
49 	rv = handle2session(hSession, &session_p);
50 	if (rv != CKR_OK)
51 		return (rv);
52 
53 	if (pMechanism == NULL) {
54 		rv = CKR_ARGUMENTS_BAD;
55 		goto clean_exit;
56 	}
57 
58 	/* Acquire the session lock */
59 	(void) pthread_mutex_lock(&session_p->session_mutex);
60 
61 	/* Check to see if digest operation is already active */
62 	if (session_p->digest.flags & CRYPTO_OPERATION_ACTIVE) {
63 		/*
64 		 * Free the memory to avoid memory leak.
65 		 * digest.context is only a flat structure.
66 		 */
67 		soft_digest_cleanup(session_p, lock_held);
68 	}
69 
70 	/*
71 	 * This active flag will remain ON until application calls either
72 	 * C_Digest or C_DigestFinal to actually obtain the value of
73 	 * the message digest.
74 	 */
75 	session_p->digest.flags = CRYPTO_OPERATION_ACTIVE;
76 
77 	(void) pthread_mutex_unlock(&session_p->session_mutex);
78 
79 	rv = soft_digest_init(session_p, pMechanism);
80 
81 	if (rv != CKR_OK) {
82 		(void) pthread_mutex_lock(&session_p->session_mutex);
83 		session_p->digest.flags &= ~CRYPTO_OPERATION_ACTIVE;
84 		/*
85 		 * Decrement the session reference count.
86 		 * We hold the session lock, and SES_REFRELE()
87 		 * will release the session lock for us.
88 		 */
89 		SES_REFRELE(session_p, lock_held);
90 		return (rv);
91 	}
92 
93 clean_exit:
94 	/*
95 	 * Decrement the session reference count.
96 	 * We do not hold the session lock.
97 	 */
98 	lock_held = B_FALSE;
99 	SES_REFRELE(session_p, lock_held);
100 	return (rv);
101 }
102 
103 
104 CK_RV
105 C_Digest(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pData, CK_ULONG ulDataLen,
106     CK_BYTE_PTR pDigest, CK_ULONG_PTR pulDigestLen)
107 {
108 
109 	CK_RV		rv;
110 	soft_session_t	*session_p;
111 	boolean_t lock_held = B_TRUE;
112 
113 	if (!softtoken_initialized)
114 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
115 
116 	/*
117 	 * Obtain the session pointer. Also, increment the session
118 	 * reference count.
119 	 */
120 	rv = handle2session(hSession, &session_p);
121 	if (rv != CKR_OK)
122 		return (rv);
123 
124 	if ((pData == NULL && ulDataLen != 0) || pulDigestLen == NULL) {
125 		rv = CKR_ARGUMENTS_BAD;
126 		goto clean_exit;
127 	}
128 
129 	/* Acquire the session lock */
130 	(void) pthread_mutex_lock(&session_p->session_mutex);
131 
132 	/* Application must call C_DigestInit before calling C_Digest */
133 	if (!(session_p->digest.flags & CRYPTO_OPERATION_ACTIVE)) {
134 		/*
135 		 * Decrement the session reference count.
136 		 * We hold the session lock, and SES_REFRELE()
137 		 * will release the session lock for us.
138 		 */
139 		SES_REFRELE(session_p, lock_held);
140 		return (CKR_OPERATION_NOT_INITIALIZED);
141 	}
142 
143 	/*
144 	 * C_Digest must be called without intervening C_DigestUpdate
145 	 * calls.
146 	 */
147 	if (session_p->digest.flags & CRYPTO_OPERATION_UPDATE) {
148 		/*
149 		 * C_Digest can not be used to terminate a multi-part
150 		 * operation, so we'll leave the active digest operation
151 		 * flag on and let the application continue with the
152 		 * digest update operation.
153 		 *
154 		 * Decrement the session reference count.
155 		 * We hold the session lock, and SES_REFRELE()
156 		 * will release the session lock for us.
157 		 */
158 		SES_REFRELE(session_p, lock_held);
159 		return (CKR_FUNCTION_FAILED);
160 	}
161 
162 	(void) pthread_mutex_unlock(&session_p->session_mutex);
163 
164 	rv = soft_digest(session_p, pData, ulDataLen, pDigest, pulDigestLen);
165 
166 	if ((rv == CKR_BUFFER_TOO_SMALL) ||
167 	    (pDigest == NULL && rv == CKR_OK)) {
168 		/*
169 		 * We will not terminate the active digest operation flag,
170 		 * when the application-supplied buffer is too small, or
171 		 * the application asks for the length of buffer to hold
172 		 * the message digest.
173 		 *
174 		 * Decrement the session reference count.
175 		 * We do not hold the session lock.
176 		 */
177 		lock_held = B_FALSE;
178 		SES_REFRELE(session_p, lock_held);
179 		return (rv);
180 	}
181 
182 clean_exit:
183 	/*
184 	 * Terminates the active digest operation.
185 	 * Application needs to call C_DigestInit again for next
186 	 * digest operation.
187 	 */
188 	(void) pthread_mutex_lock(&session_p->session_mutex);
189 
190 	soft_digest_cleanup(session_p, lock_held);
191 
192 	/*
193 	 * Decrement the session reference count.
194 	 * We hold the session lock, and SES_REFRELE()
195 	 * will release the session lock for us.
196 	 */
197 	SES_REFRELE(session_p, lock_held);
198 
199 	return (rv);
200 }
201 
202 
203 CK_RV
204 C_DigestUpdate(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pPart,
205     CK_ULONG ulPartLen)
206 {
207 
208 	CK_RV		rv;
209 	soft_session_t	*session_p;
210 	boolean_t lock_held = B_TRUE;
211 
212 	if (!softtoken_initialized)
213 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
214 
215 	/*
216 	 * Obtain the session pointer. Also, increment the session
217 	 * reference count.
218 	 */
219 	rv = handle2session(hSession, &session_p);
220 	if (rv != CKR_OK)
221 		return (rv);
222 
223 	if (pPart == NULL) {
224 		rv = CKR_ARGUMENTS_BAD;
225 		goto clean_exit;
226 	}
227 
228 	/* Acquire the session lock */
229 	(void) pthread_mutex_lock(&session_p->session_mutex);
230 
231 	/*
232 	 * Application must call C_DigestInit before calling
233 	 * C_DigestUpdate.
234 	 */
235 	if (!(session_p->digest.flags & CRYPTO_OPERATION_ACTIVE)) {
236 		/*
237 		 * Decrement the session reference count.
238 		 * We hold the session lock, and SES_REFRELE()
239 		 * will release the session lock for us.
240 		 */
241 		SES_REFRELE(session_p, lock_held);
242 		return (CKR_OPERATION_NOT_INITIALIZED);
243 	}
244 
245 	/* Set update flag to protect C_Digest */
246 	session_p->digest.flags |= CRYPTO_OPERATION_UPDATE;
247 
248 	(void) pthread_mutex_unlock(&session_p->session_mutex);
249 
250 	rv = soft_digest_update(session_p, pPart, ulPartLen);
251 
252 	if (rv == CKR_OK) {
253 		/*
254 		 * Decrement the session reference count.
255 		 * We do not hold the session lock.
256 		 */
257 		lock_held = B_FALSE;
258 		SES_REFRELE(session_p, lock_held);
259 		return (CKR_OK);
260 	}
261 
262 clean_exit:
263 	/*
264 	 * After an error occurred, terminate the current digest
265 	 * operation by resetting the active and update flags.
266 	 */
267 	(void) pthread_mutex_lock(&session_p->session_mutex);
268 
269 	soft_digest_cleanup(session_p, lock_held);
270 
271 	/*
272 	 * Decrement the session reference count.
273 	 * We hold the session lock, and SES_REFRELE()
274 	 * will release the session lock for us.
275 	 */
276 	SES_REFRELE(session_p, lock_held);
277 
278 	return (rv);
279 }
280 
281 
282 CK_RV
283 C_DigestKey(CK_SESSION_HANDLE hSession, CK_OBJECT_HANDLE hKey)
284 {
285 
286 	CK_RV		rv;
287 	soft_session_t	*session_p;
288 	soft_object_t	*key_p;
289 	boolean_t lock_held = B_TRUE;
290 
291 	if (!softtoken_initialized)
292 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
293 
294 	/*
295 	 * Obtain the session pointer. Also, increment the session
296 	 * reference count.
297 	 */
298 	rv = handle2session(hSession, &session_p);
299 	if (rv != CKR_OK)
300 		return (rv);
301 
302 	/* Obtain the object pointer. */
303 	HANDLE2OBJECT(hKey, key_p, rv);
304 	if (rv != CKR_OK)
305 		goto clean_exit;
306 
307 	(void) pthread_mutex_lock(&session_p->session_mutex);
308 
309 	/*
310 	 * Application must call C_DigestInit before calling
311 	 * C_DigestKey.
312 	 */
313 	if (!(session_p->digest.flags & CRYPTO_OPERATION_ACTIVE)) {
314 		/*
315 		 * Decrement the session reference count.
316 		 * We hold the session lock, and SES_REFRELE()
317 		 * will release the session lock for us.
318 		 */
319 		OBJ_REFRELE(key_p);
320 		SES_REFRELE(session_p, lock_held);
321 		return (CKR_OPERATION_NOT_INITIALIZED);
322 	}
323 
324 	/*
325 	 * Remember the fact that a key was thrown into the mix, so that
326 	 * C_DigestFinal bzero()'s the digest context before freeing it.
327 	 */
328 	session_p->digest.flags |= (CRYPTO_KEY_DIGESTED |
329 	    CRYPTO_OPERATION_UPDATE);
330 
331 	(void) pthread_mutex_unlock(&session_p->session_mutex);
332 
333 	rv = soft_digest_key(session_p, key_p);
334 
335 	if (rv == CKR_OK) {
336 		/*
337 		 * Decrement the session reference count.
338 		 * We do not hold the session lock.
339 		 */
340 		lock_held = B_FALSE;
341 		OBJ_REFRELE(key_p);
342 		SES_REFRELE(session_p, lock_held);
343 		return (CKR_OK);
344 	}
345 
346 	OBJ_REFRELE(key_p);
347 clean_exit:
348 	/*
349 	 * After an error occurred, terminate the current digest
350 	 * operation by resetting the active and update flags.
351 	 */
352 	(void) pthread_mutex_lock(&session_p->session_mutex);
353 
354 	soft_digest_cleanup(session_p, lock_held);
355 
356 	/*
357 	 * Decrement the session reference count.
358 	 * We hold the session lock, and SES_REFRELE()
359 	 * will release the session lock for us.
360 	 */
361 	SES_REFRELE(session_p, lock_held);
362 	return (rv);
363 }
364 
365 
366 CK_RV
367 C_DigestFinal(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pDigest,
368     CK_ULONG_PTR pulDigestLen)
369 {
370 
371 	CK_RV		rv;
372 	soft_session_t	*session_p;
373 	boolean_t lock_held = B_TRUE;
374 
375 	if (!softtoken_initialized)
376 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
377 
378 	/*
379 	 * Obtain the session pointer. Also, increment the session
380 	 * reference count.
381 	 */
382 	rv = handle2session(hSession, &session_p);
383 	if (rv != CKR_OK)
384 		return (rv);
385 
386 	if (pulDigestLen == NULL) {
387 		rv = CKR_ARGUMENTS_BAD;
388 		goto clean_exit;
389 	}
390 
391 	/* Acquire the session lock */
392 	(void) pthread_mutex_lock(&session_p->session_mutex);
393 
394 	/*
395 	 * Application must call C_DigestInit before calling
396 	 * C_DigestFinal.
397 	 */
398 	if (!(session_p->digest.flags & CRYPTO_OPERATION_ACTIVE)) {
399 		/*
400 		 * Decrement the session reference count.
401 		 * We hold the session lock, and SES_REFRELE()
402 		 * will release the session lock for us.
403 		 */
404 		SES_REFRELE(session_p, lock_held);
405 		return (CKR_OPERATION_NOT_INITIALIZED);
406 	}
407 
408 	(void) pthread_mutex_unlock(&session_p->session_mutex);
409 
410 	rv = soft_digest_final(session_p, pDigest, pulDigestLen);
411 
412 	if ((rv == CKR_BUFFER_TOO_SMALL) ||
413 	    (pDigest == NULL && rv == CKR_OK)) {
414 		/*
415 		 * We will not terminate the active digest operation flag,
416 		 * when the application-supplied buffer is too small, or
417 		 * the application asks for the length of buffer to hold
418 		 * the message digest.
419 		 *
420 		 * Decrement the session reference count.
421 		 * We do not hold the session lock.
422 		 */
423 		lock_held = B_FALSE;
424 		SES_REFRELE(session_p, lock_held);
425 		return (rv);
426 	}
427 
428 clean_exit:
429 	/* Terminates the active digest operation */
430 	(void) pthread_mutex_lock(&session_p->session_mutex);
431 
432 	soft_digest_cleanup(session_p, lock_held);
433 
434 	/*
435 	 * Decrement the session reference count.
436 	 * We hold the session lock, and SES_REFRELE()
437 	 * will release the session lock for us.
438 	 */
439 	SES_REFRELE(session_p, lock_held);
440 
441 	return (rv);
442 }
443