xref: /titanic_41/usr/src/lib/pkcs11/pkcs11_softtoken/common/softDigestUtil.c (revision 657a8c206b913d1ee578fd725f0b25eca5b77253)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <strings.h>
27 #include <md5.h>
28 #include <pthread.h>
29 #include <stdlib.h>
30 #include <sys/sha1.h>
31 #include <sys/sha2.h>
32 #include <sys/types.h>
33 #include <security/cryptoki.h>
34 #include "softGlobal.h"
35 #include "softOps.h"
36 #include "softSession.h"
37 #include "softObject.h"
38 
39 
40 /*
41  * soft_digest_init()
42  *
43  * Arguments:
44  *	session_p:	pointer to soft_session_t struct
45  *	pMechanism:	pointer to CK_MECHANISM struct provided by application
46  *
47  * Description:
48  *	called by C_DigestInit(). This function allocates space for
49  *  	context, then calls the corresponding software provided digest
50  *	init routine based on the mechanism.
51  *
52  * Returns:
53  *	CKR_OK: success
54  *	CKR_HOST_MEMORY: run out of system memory
55  *	CKR_MECHANISM_INVALID: invalid mechanism type
56  */
57 CK_RV
58 soft_digest_init(soft_session_t *session_p, CK_MECHANISM_PTR pMechanism)
59 {
60 
61 	switch (pMechanism->mechanism) {
62 
63 	case CKM_MD5:
64 		(void) pthread_mutex_lock(&session_p->session_mutex);
65 
66 		session_p->digest.context = malloc(sizeof (MD5_CTX));
67 
68 		if (session_p->digest.context == NULL) {
69 			(void) pthread_mutex_unlock(&session_p->session_mutex);
70 			return (CKR_HOST_MEMORY);
71 		}
72 
73 		session_p->digest.mech.mechanism = CKM_MD5;
74 		(void) pthread_mutex_unlock(&session_p->session_mutex);
75 
76 		MD5Init((MD5_CTX *)session_p->digest.context);
77 
78 		break;
79 
80 	case CKM_SHA_1:
81 
82 		(void) pthread_mutex_lock(&session_p->session_mutex);
83 
84 		session_p->digest.context = malloc(sizeof (SHA1_CTX));
85 
86 		if (session_p->digest.context == NULL) {
87 			(void) pthread_mutex_unlock(&session_p->session_mutex);
88 			return (CKR_HOST_MEMORY);
89 		}
90 
91 		session_p->digest.mech.mechanism = CKM_SHA_1;
92 		session_p->digest.mech.pParameter = pMechanism->pParameter;
93 		session_p->digest.mech.ulParameterLen =
94 		    pMechanism->ulParameterLen;
95 		(void) pthread_mutex_unlock(&session_p->session_mutex);
96 
97 		SHA1Init((SHA1_CTX *)session_p->digest.context);
98 
99 		break;
100 
101 	case CKM_SHA256:
102 	case CKM_SHA384:
103 	case CKM_SHA512:
104 
105 		(void) pthread_mutex_lock(&session_p->session_mutex);
106 
107 		session_p->digest.context = malloc(sizeof (SHA2_CTX));
108 
109 		if (session_p->digest.context == NULL) {
110 			(void) pthread_mutex_unlock(&session_p->session_mutex);
111 			return (CKR_HOST_MEMORY);
112 		}
113 
114 		switch (pMechanism->mechanism) {
115 		case CKM_SHA256:
116 			session_p->digest.mech.mechanism = CKM_SHA256;
117 			(void) pthread_mutex_unlock(&session_p->session_mutex);
118 			SHA2Init(SHA256,
119 			    (SHA2_CTX *)session_p->digest.context);
120 			break;
121 
122 		case CKM_SHA384:
123 			session_p->digest.mech.mechanism = CKM_SHA384;
124 			(void) pthread_mutex_unlock(&session_p->session_mutex);
125 			SHA2Init(SHA384,
126 			    (SHA2_CTX *)session_p->digest.context);
127 			break;
128 
129 		case CKM_SHA512:
130 			session_p->digest.mech.mechanism = CKM_SHA512;
131 			(void) pthread_mutex_unlock(&session_p->session_mutex);
132 			SHA2Init(SHA512,
133 			    (SHA2_CTX *)session_p->digest.context);
134 			break;
135 		}
136 		break;
137 
138 	default:
139 		return (CKR_MECHANISM_INVALID);
140 	}
141 
142 	return (CKR_OK);
143 }
144 
145 
146 /*
147  * soft_digest_common()
148  *
149  * Arguments:
150  *      session_p:	pointer to soft_session_t struct
151  *	pData:		pointer to the input data to be digested
152  *	ulDataLen:	length of the input data
153  *	pDigest:	pointer to the output data after digesting
154  *	pulDigestLen:	length of the output data
155  *
156  * Description:
157  *      called by soft_digest() or soft_digest_final(). This function
158  *      determines the length of output buffer and calls the corresponding
159  *	software provided digest routine based on the mechanism.
160  *
161  * Returns:
162  *      CKR_OK: success
163  *      CKR_MECHANISM_INVALID: invalid mechanism type
164  *      CKR_BUFFER_TOO_SMALL: the output buffer provided by application
165  *			      is too small
166  */
167 CK_RV
168 soft_digest_common(soft_session_t *session_p, CK_BYTE_PTR pData,
169 	CK_ULONG ulDataLen, CK_BYTE_PTR pDigest, CK_ULONG_PTR pulDigestLen)
170 {
171 
172 	CK_ULONG digestLen = 0;
173 	size_t len = 0;
174 
175 	/*
176 	 * Determine the output data length based on the mechanism
177 	 */
178 	switch (session_p->digest.mech.mechanism) {
179 
180 	case CKM_MD5:
181 		digestLen = 16;
182 		break;
183 
184 	case CKM_SHA_1:
185 		digestLen = 20;
186 		break;
187 
188 	case CKM_SHA256:
189 		digestLen = 32;
190 		break;
191 
192 	case CKM_SHA384:
193 		digestLen = 48;
194 		break;
195 
196 	case CKM_SHA512:
197 		digestLen = 64;
198 		break;
199 
200 	default:
201 		return (CKR_MECHANISM_INVALID);
202 	}
203 
204 	if (pDigest == NULL) {
205 		/*
206 		 * Application only wants to know the length of the
207 		 * buffer needed to hold the message digest.
208 		 */
209 		*pulDigestLen = digestLen;
210 		return (CKR_OK);
211 	}
212 
213 	if (*pulDigestLen < digestLen) {
214 		/*
215 		 * Application provides buffer too small to hold the
216 		 * digest message. Return the length of buffer needed
217 		 * to the application.
218 		 */
219 		*pulDigestLen = digestLen;
220 		return (CKR_BUFFER_TOO_SMALL);
221 	}
222 
223 	/*
224 	 * Call the corresponding system provided software digest routine.
225 	 * If the soft_digest_common() is called by soft_digest_final()
226 	 * the pData is NULL, and the ulDataLen is zero.
227 	 */
228 	switch (session_p->digest.mech.mechanism) {
229 
230 	case CKM_MD5:
231 		if (pData != NULL) {
232 			/*
233 			 * this is called by soft_digest()
234 			 */
235 #ifdef	__sparcv9
236 			MD5Update((MD5_CTX *)session_p->digest.context,
237 			    /* LINTED */
238 			    pData, (uint_t)ulDataLen);
239 #else	/* !__sparcv9 */
240 			MD5Update((MD5_CTX *)session_p->digest.context,
241 			    pData, ulDataLen);
242 #endif	/* __sparcv9 */
243 			MD5Final(pDigest, (MD5_CTX *)session_p->digest.context);
244 		} else {
245 			/*
246 			 * this is called by soft_digest_final()
247 			 */
248 			MD5Final(pDigest, (MD5_CTX *)session_p->digest.context);
249 			len = sizeof (MD5_CTX);
250 		}
251 		break;
252 
253 	case CKM_SHA_1:
254 		if (pData != NULL) {
255 			/*
256 			 * this is called by soft_digest()
257 			 */
258 
259 #ifdef	__sparcv9
260 			SHA1Update((SHA1_CTX *)session_p->digest.context,
261 			    /* LINTED */
262 			    pData, (uint32_t)ulDataLen);
263 #else	/* !__sparcv9 */
264 			SHA1Update((SHA1_CTX *)session_p->digest.context,
265 			    pData, ulDataLen);
266 #endif	/* __sparcv9 */
267 			SHA1Final(pDigest,
268 			    (SHA1_CTX *)session_p->digest.context);
269 		} else {
270 			/*
271 			 * this is called by soft_digest_final()
272 			 */
273 			SHA1Final(pDigest,
274 			    (SHA1_CTX *)session_p->digest.context);
275 			len = sizeof (SHA1_CTX);
276 		}
277 		break;
278 	case CKM_SHA256:
279 	case CKM_SHA384:
280 	case CKM_SHA512:
281 		if (pData != NULL) {
282 			/*
283 			 * this is called by soft_digest()
284 			 */
285 
286 			SHA2Update((SHA2_CTX *)session_p->digest.context,
287 			    pData, ulDataLen);
288 
289 			SHA2Final(pDigest,
290 			    (SHA2_CTX *)session_p->digest.context);
291 		} else {
292 			/*
293 			 * this is called by soft_digest_final()
294 			 */
295 			SHA2Final(pDigest,
296 			    (SHA2_CTX *)session_p->digest.context);
297 			len = sizeof (SHA2_CTX);
298 		}
299 
300 		break;
301 	}
302 
303 	/* Paranoia on behalf of C_DigestKey callers: bzero the context */
304 	if (session_p->digest.flags & CRYPTO_KEY_DIGESTED) {
305 		bzero(session_p->digest.context, len);
306 		session_p->digest.flags &= ~CRYPTO_KEY_DIGESTED;
307 	}
308 	*pulDigestLen = digestLen;
309 	(void) pthread_mutex_lock(&session_p->session_mutex);
310 	free(session_p->digest.context);
311 	session_p->digest.context = NULL;
312 	(void) pthread_mutex_unlock(&session_p->session_mutex);
313 
314 	return (CKR_OK);
315 }
316 
317 
318 /*
319  * soft_digest()
320  *
321  * Arguments:
322  *      session_p:	pointer to soft_session_t struct
323  *      pData:		pointer to the input data to be digested
324  *      ulDataLen:	length of the input data
325  *      pDigest:	pointer to the output data after digesting
326  *      pulDigestLen:	length of the output data
327  *
328  * Description:
329  *      called by C_Digest(). This function calls soft_digest_common().
330  *
331  * Returns:
332  *      see return values in soft_digest_common().
333  */
334 CK_RV
335 soft_digest(soft_session_t *session_p, CK_BYTE_PTR pData, CK_ULONG ulDataLen,
336 	CK_BYTE_PTR pDigest, CK_ULONG_PTR pulDigestLen)
337 {
338 
339 	return (soft_digest_common(session_p, pData, ulDataLen,
340 	    pDigest, pulDigestLen));
341 }
342 
343 
344 /*
345  * soft_digest_update()
346  *
347  * Arguments:
348  *      session_p:	pointer to soft_session_t struct
349  *      pPart:		pointer to the input data to be digested
350  *      ulPartLen:	length of the input data
351  *
352  * Description:
353  *      called by C_DigestUpdate(). This function calls the corresponding
354  *	software provided digest update routine based on the mechanism.
355  *
356  * Returns:
357  *      CKR_OK: success
358  *      CKR_MECHANISM_INVALID: invalid MECHANISM type.
359  */
360 CK_RV
361 soft_digest_update(soft_session_t *session_p, CK_BYTE_PTR pPart,
362 	CK_ULONG ulPartLen)
363 {
364 
365 	switch (session_p->digest.mech.mechanism) {
366 
367 	case CKM_MD5:
368 #ifdef	__sparcv9
369 		MD5Update((MD5_CTX *)session_p->digest.context,
370 		    /* LINTED */
371 		    pPart, (uint_t)ulPartLen);
372 #else	/* !__sparcv9 */
373 		MD5Update((MD5_CTX *)session_p->digest.context,
374 		    pPart, ulPartLen);
375 #endif	/* __sparcv9 */
376 		break;
377 
378 	case CKM_SHA_1:
379 #ifdef	__sparcv9
380 		SHA1Update((SHA1_CTX *)session_p->digest.context,
381 		    /* LINTED */
382 		    pPart, (uint32_t)ulPartLen);
383 #else	/* !__sparcv9 */
384 		SHA1Update((SHA1_CTX *)session_p->digest.context,
385 		    pPart, ulPartLen);
386 #endif	/* __sparcv9 */
387 		break;
388 
389 	case CKM_SHA256:
390 	case CKM_SHA384:
391 	case CKM_SHA512:
392 		SHA2Update((SHA2_CTX *)session_p->digest.context,
393 		    pPart, ulPartLen);
394 		break;
395 
396 	default:
397 		return (CKR_MECHANISM_INVALID);
398 	}
399 
400 	return (CKR_OK);
401 }
402 
403 
404 /*
405  * soft_digest_final()
406  *
407  * Arguments:
408  *      session_p:	pointer to soft_session_t struct
409  *      pDigest:	pointer to the output data after digesting
410  *      pulDigestLen:	length of the output data
411  *
412  * Description:
413  *      called by C_DigestFinal(). This function calls soft_digest_common().
414  *
415  * Returns:
416  *	see return values in soft_digest_common().
417  */
418 CK_RV
419 soft_digest_final(soft_session_t *session_p, CK_BYTE_PTR pDigest,
420 	CK_ULONG_PTR pulDigestLen)
421 {
422 
423 	return (soft_digest_common(session_p, NULL, 0,
424 	    pDigest, pulDigestLen));
425 }
426 
427 /*
428  * Perform digest init operation internally for the support of
429  * CKM_MD5_RSA_PKCS, CKM_SHA1_RSA_PKCS, CKM_SHA1_KEY_DERIVATION
430  * and CKM_MD5_KEY_DERIVATION mechanisms.
431  *
432  * This function is called with the session being held, and without
433  * its mutex taken.
434  */
435 CK_RV
436 soft_digest_init_internal(soft_session_t *session_p, CK_MECHANISM_PTR
437 	pMechanism)
438 {
439 
440 	CK_RV rv;
441 
442 	(void) pthread_mutex_lock(&session_p->session_mutex);
443 
444 	/* Check to see if digest operation is already active */
445 	if (session_p->digest.flags & CRYPTO_OPERATION_ACTIVE) {
446 		(void) pthread_mutex_unlock(&session_p->session_mutex);
447 		return (CKR_OPERATION_ACTIVE);
448 	}
449 
450 	session_p->digest.flags = CRYPTO_OPERATION_ACTIVE;
451 
452 	(void) pthread_mutex_unlock(&session_p->session_mutex);
453 
454 	rv = soft_digest_init(session_p, pMechanism);
455 
456 	if (rv != CKR_OK) {
457 		(void) pthread_mutex_lock(&session_p->session_mutex);
458 		session_p->digest.flags &= ~CRYPTO_OPERATION_ACTIVE;
459 		(void) pthread_mutex_unlock(&session_p->session_mutex);
460 	}
461 
462 	return (rv);
463 }
464 
465 /*
466  * Call soft_digest_update() function with the value of a secret key.
467  */
468 CK_RV
469 soft_digest_key(soft_session_t *session_p, soft_object_t *key_p)
470 {
471 
472 	CK_RV rv;
473 
474 	/* Only secret key is allowed to be digested */
475 	if (key_p->class != CKO_SECRET_KEY)
476 		return (CKR_KEY_INDIGESTIBLE);
477 
478 	if ((OBJ_SEC_VALUE(key_p) == NULL) ||
479 	    (OBJ_SEC_VALUE_LEN(key_p) == 0))
480 		return (CKR_KEY_SIZE_RANGE);
481 
482 	rv = soft_digest_update(session_p, OBJ_SEC_VALUE(key_p),
483 	    OBJ_SEC_VALUE_LEN(key_p));
484 
485 	return (rv);
486 
487 }
488 
489 /*
490  * This function releases allocated digest context. The caller
491  * may (lock_held == B_TRUE) or may not (lock_held == B_FALSE)
492  * hold a session mutex.
493  */
494 void
495 soft_digest_cleanup(soft_session_t *session_p, boolean_t lock_held)
496 {
497 	boolean_t lock_true = B_TRUE;
498 
499 	if (!lock_held)
500 		(void) pthread_mutex_lock(&session_p->session_mutex);
501 
502 	if (session_p->digest.context != NULL) {
503 		free(session_p->digest.context);
504 		session_p->digest.context = NULL;
505 	}
506 
507 	session_p->digest.flags = 0;
508 
509 	if (!lock_held)
510 		SES_REFRELE(session_p, lock_true);
511 
512 }
513