xref: /titanic_50/usr/src/lib/pkcs11/pkcs11_softtoken/common/softDigestUtil.c (revision c77a61a72b5ecdc507d6cf104142edd371a16c84)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 #include <strings.h>
30 #include <md5.h>
31 #include <pthread.h>
32 #include <stdlib.h>
33 #include <sys/sha1.h>
34 #include <sys/sha2.h>
35 #include <sys/types.h>
36 #include <security/cryptoki.h>
37 #include "softGlobal.h"
38 #include "softOps.h"
39 #include "softSession.h"
40 #include "softObject.h"
41 
42 
43 /*
44  * soft_digest_init()
45  *
46  * Arguments:
47  *	session_p:	pointer to soft_session_t struct
48  *	pMechanism:	pointer to CK_MECHANISM struct provided by application
49  *
50  * Description:
51  *	called by C_DigestInit(). This function allocates space for
52  *  	context, then calls the corresponding software provided digest
53  *	init routine based on the mechanism.
54  *
55  * Returns:
56  *	CKR_OK: success
57  *	CKR_HOST_MEMORY: run out of system memory
58  *	CKR_MECHANISM_INVALID: invalid mechanism type
59  */
60 CK_RV
61 soft_digest_init(soft_session_t *session_p, CK_MECHANISM_PTR pMechanism)
62 {
63 
64 	switch (pMechanism->mechanism) {
65 
66 	case CKM_MD5:
67 		(void) pthread_mutex_lock(&session_p->session_mutex);
68 
69 		session_p->digest.context = malloc(sizeof (MD5_CTX));
70 
71 		if (session_p->digest.context == NULL) {
72 			(void) pthread_mutex_unlock(&session_p->session_mutex);
73 			return (CKR_HOST_MEMORY);
74 		}
75 
76 		session_p->digest.mech.mechanism = CKM_MD5;
77 		(void) pthread_mutex_unlock(&session_p->session_mutex);
78 
79 		MD5Init((MD5_CTX *)session_p->digest.context);
80 
81 		break;
82 
83 	case CKM_SHA_1:
84 
85 		(void) pthread_mutex_lock(&session_p->session_mutex);
86 
87 		session_p->digest.context = malloc(sizeof (SHA1_CTX));
88 
89 		if (session_p->digest.context == NULL) {
90 			(void) pthread_mutex_unlock(&session_p->session_mutex);
91 			return (CKR_HOST_MEMORY);
92 		}
93 
94 		session_p->digest.mech.mechanism = CKM_SHA_1;
95 		session_p->digest.mech.pParameter = pMechanism->pParameter;
96 		session_p->digest.mech.ulParameterLen =
97 		    pMechanism->ulParameterLen;
98 		(void) pthread_mutex_unlock(&session_p->session_mutex);
99 
100 		SHA1Init((SHA1_CTX *)session_p->digest.context);
101 
102 		break;
103 
104 	case CKM_SHA256:
105 	case CKM_SHA384:
106 	case CKM_SHA512:
107 
108 		(void) pthread_mutex_lock(&session_p->session_mutex);
109 
110 		session_p->digest.context = malloc(sizeof (SHA2_CTX));
111 
112 		if (session_p->digest.context == NULL) {
113 			(void) pthread_mutex_unlock(&session_p->session_mutex);
114 			return (CKR_HOST_MEMORY);
115 		}
116 
117 		switch (pMechanism->mechanism) {
118 		case CKM_SHA256:
119 			session_p->digest.mech.mechanism = CKM_SHA256;
120 			(void) pthread_mutex_unlock(&session_p->session_mutex);
121 			SHA2Init(SHA256,
122 			    (SHA2_CTX *)session_p->digest.context);
123 			break;
124 
125 		case CKM_SHA384:
126 			session_p->digest.mech.mechanism = CKM_SHA384;
127 			(void) pthread_mutex_unlock(&session_p->session_mutex);
128 			SHA2Init(SHA384,
129 			    (SHA2_CTX *)session_p->digest.context);
130 			break;
131 
132 		case CKM_SHA512:
133 			session_p->digest.mech.mechanism = CKM_SHA512;
134 			(void) pthread_mutex_unlock(&session_p->session_mutex);
135 			SHA2Init(SHA512,
136 			    (SHA2_CTX *)session_p->digest.context);
137 			break;
138 		}
139 		break;
140 
141 	default:
142 		return (CKR_MECHANISM_INVALID);
143 	}
144 
145 	return (CKR_OK);
146 }
147 
148 
149 /*
150  * soft_digest_common()
151  *
152  * Arguments:
153  *      session_p:	pointer to soft_session_t struct
154  *	pData:		pointer to the input data to be digested
155  *	ulDataLen:	length of the input data
156  *	pDigest:	pointer to the output data after digesting
157  *	pulDigestLen:	length of the output data
158  *
159  * Description:
160  *      called by soft_digest() or soft_digest_final(). This function
161  *      determines the length of output buffer and calls the corresponding
162  *	software provided digest routine based on the mechanism.
163  *
164  * Returns:
165  *      CKR_OK: success
166  *      CKR_MECHANISM_INVALID: invalid mechanism type
167  *      CKR_BUFFER_TOO_SMALL: the output buffer provided by application
168  *			      is too small
169  */
170 CK_RV
171 soft_digest_common(soft_session_t *session_p, CK_BYTE_PTR pData,
172 	CK_ULONG ulDataLen, CK_BYTE_PTR pDigest, CK_ULONG_PTR pulDigestLen)
173 {
174 
175 	CK_ULONG digestLen = 0;
176 	size_t len = 0;
177 
178 	/*
179 	 * Determine the output data length based on the mechanism
180 	 */
181 	switch (session_p->digest.mech.mechanism) {
182 
183 	case CKM_MD5:
184 		digestLen = 16;
185 		break;
186 
187 	case CKM_SHA_1:
188 		digestLen = 20;
189 		break;
190 
191 	case CKM_SHA256:
192 		digestLen = 32;
193 		break;
194 
195 	case CKM_SHA384:
196 		digestLen = 48;
197 		break;
198 
199 	case CKM_SHA512:
200 		digestLen = 64;
201 		break;
202 
203 	default:
204 		return (CKR_MECHANISM_INVALID);
205 	}
206 
207 	if (pDigest == NULL) {
208 		/*
209 		 * Application only wants to know the length of the
210 		 * buffer needed to hold the message digest.
211 		 */
212 		*pulDigestLen = digestLen;
213 		return (CKR_OK);
214 	}
215 
216 	if (*pulDigestLen < digestLen) {
217 		/*
218 		 * Application provides buffer too small to hold the
219 		 * digest message. Return the length of buffer needed
220 		 * to the application.
221 		 */
222 		*pulDigestLen = digestLen;
223 		return (CKR_BUFFER_TOO_SMALL);
224 	}
225 
226 	/*
227 	 * Call the corresponding system provided software digest routine.
228 	 * If the soft_digest_common() is called by soft_digest_final()
229 	 * the pData is NULL, and the ulDataLen is zero.
230 	 */
231 	switch (session_p->digest.mech.mechanism) {
232 
233 	case CKM_MD5:
234 		if (pData != NULL) {
235 			/*
236 			 * this is called by soft_digest()
237 			 */
238 #ifdef	__sparcv9
239 			MD5Update((MD5_CTX *)session_p->digest.context,
240 			    /* LINTED */
241 			    pData, (uint_t)ulDataLen);
242 #else	/* !__sparcv9 */
243 			MD5Update((MD5_CTX *)session_p->digest.context,
244 			    pData, ulDataLen);
245 #endif	/* __sparcv9 */
246 			MD5Final(pDigest, (MD5_CTX *)session_p->digest.context);
247 		} else {
248 			/*
249 			 * this is called by soft_digest_final()
250 			 */
251 			MD5Final(pDigest, (MD5_CTX *)session_p->digest.context);
252 			len = sizeof (MD5_CTX);
253 		}
254 		break;
255 
256 	case CKM_SHA_1:
257 		if (pData != NULL) {
258 			/*
259 			 * this is called by soft_digest()
260 			 */
261 
262 #ifdef	__sparcv9
263 			SHA1Update((SHA1_CTX *)session_p->digest.context,
264 			    /* LINTED */
265 			    pData, (uint32_t)ulDataLen);
266 #else	/* !__sparcv9 */
267 			SHA1Update((SHA1_CTX *)session_p->digest.context,
268 			    pData, ulDataLen);
269 #endif	/* __sparcv9 */
270 			SHA1Final(pDigest,
271 			    (SHA1_CTX *)session_p->digest.context);
272 		} else {
273 			/*
274 			 * this is called by soft_digest_final()
275 			 */
276 			SHA1Final(pDigest,
277 			    (SHA1_CTX *)session_p->digest.context);
278 			len = sizeof (SHA1_CTX);
279 		}
280 		break;
281 	case CKM_SHA256:
282 	case CKM_SHA384:
283 	case CKM_SHA512:
284 		if (pData != NULL) {
285 			/*
286 			 * this is called by soft_digest()
287 			 */
288 
289 			SHA2Update((SHA2_CTX *)session_p->digest.context,
290 			    pData, ulDataLen);
291 
292 			SHA2Final(pDigest,
293 			    (SHA2_CTX *)session_p->digest.context);
294 		} else {
295 			/*
296 			 * this is called by soft_digest_final()
297 			 */
298 			SHA2Final(pDigest,
299 			    (SHA2_CTX *)session_p->digest.context);
300 			len = sizeof (SHA2_CTX);
301 		}
302 
303 		break;
304 	}
305 
306 	/* Paranoia on behalf of C_DigestKey callers: bzero the context */
307 	if (session_p->digest.flags & CRYPTO_KEY_DIGESTED) {
308 		bzero(session_p->digest.context, len);
309 		session_p->digest.flags &= ~CRYPTO_KEY_DIGESTED;
310 	}
311 	*pulDigestLen = digestLen;
312 	(void) pthread_mutex_lock(&session_p->session_mutex);
313 	free(session_p->digest.context);
314 	session_p->digest.context = NULL;
315 	(void) pthread_mutex_unlock(&session_p->session_mutex);
316 
317 	return (CKR_OK);
318 }
319 
320 
321 /*
322  * soft_digest()
323  *
324  * Arguments:
325  *      session_p:	pointer to soft_session_t struct
326  *      pData:		pointer to the input data to be digested
327  *      ulDataLen:	length of the input data
328  *      pDigest:	pointer to the output data after digesting
329  *      pulDigestLen:	length of the output data
330  *
331  * Description:
332  *      called by C_Digest(). This function calls soft_digest_common().
333  *
334  * Returns:
335  *      see return values in soft_digest_common().
336  */
337 CK_RV
338 soft_digest(soft_session_t *session_p, CK_BYTE_PTR pData, CK_ULONG ulDataLen,
339 	CK_BYTE_PTR pDigest, CK_ULONG_PTR pulDigestLen)
340 {
341 
342 	return (soft_digest_common(session_p, pData, ulDataLen,
343 	    pDigest, pulDigestLen));
344 }
345 
346 
347 /*
348  * soft_digest_update()
349  *
350  * Arguments:
351  *      session_p:	pointer to soft_session_t struct
352  *      pPart:		pointer to the input data to be digested
353  *      ulPartLen:	length of the input data
354  *
355  * Description:
356  *      called by C_DigestUpdate(). This function calls the corresponding
357  *	software provided digest update routine based on the mechanism.
358  *
359  * Returns:
360  *      CKR_OK: success
361  *      CKR_MECHANISM_INVALID: invalid MECHANISM type.
362  */
363 CK_RV
364 soft_digest_update(soft_session_t *session_p, CK_BYTE_PTR pPart,
365 	CK_ULONG ulPartLen)
366 {
367 
368 	switch (session_p->digest.mech.mechanism) {
369 
370 	case CKM_MD5:
371 #ifdef	__sparcv9
372 		MD5Update((MD5_CTX *)session_p->digest.context,
373 		    /* LINTED */
374 		    pPart, (uint_t)ulPartLen);
375 #else	/* !__sparcv9 */
376 		MD5Update((MD5_CTX *)session_p->digest.context,
377 		    pPart, ulPartLen);
378 #endif	/* __sparcv9 */
379 		break;
380 
381 	case CKM_SHA_1:
382 #ifdef	__sparcv9
383 		SHA1Update((SHA1_CTX *)session_p->digest.context,
384 		    /* LINTED */
385 		    pPart, (uint32_t)ulPartLen);
386 #else	/* !__sparcv9 */
387 		SHA1Update((SHA1_CTX *)session_p->digest.context,
388 		    pPart, ulPartLen);
389 #endif	/* __sparcv9 */
390 		break;
391 
392 	case CKM_SHA256:
393 	case CKM_SHA384:
394 	case CKM_SHA512:
395 		SHA2Update((SHA2_CTX *)session_p->digest.context,
396 		    pPart, ulPartLen);
397 		break;
398 
399 	default:
400 		return (CKR_MECHANISM_INVALID);
401 	}
402 
403 	return (CKR_OK);
404 }
405 
406 
407 /*
408  * soft_digest_final()
409  *
410  * Arguments:
411  *      session_p:	pointer to soft_session_t struct
412  *      pDigest:	pointer to the output data after digesting
413  *      pulDigestLen:	length of the output data
414  *
415  * Description:
416  *      called by C_DigestFinal(). This function calls soft_digest_common().
417  *
418  * Returns:
419  *	see return values in soft_digest_common().
420  */
421 CK_RV
422 soft_digest_final(soft_session_t *session_p, CK_BYTE_PTR pDigest,
423 	CK_ULONG_PTR pulDigestLen)
424 {
425 
426 	return (soft_digest_common(session_p, NULL, 0,
427 	    pDigest, pulDigestLen));
428 }
429 
430 /*
431  * Perform digest init operation internally for the support of
432  * CKM_MD5_RSA_PKCS, CKM_SHA1_RSA_PKCS, CKM_SHA1_KEY_DERIVATION
433  * and CKM_MD5_KEY_DERIVATION mechanisms.
434  *
435  * This function is called with the session being held, and without
436  * its mutex taken.
437  */
438 CK_RV
439 soft_digest_init_internal(soft_session_t *session_p, CK_MECHANISM_PTR
440 	pMechanism)
441 {
442 
443 	CK_RV rv;
444 
445 	(void) pthread_mutex_lock(&session_p->session_mutex);
446 
447 	/* Check to see if digest operation is already active */
448 	if (session_p->digest.flags & CRYPTO_OPERATION_ACTIVE) {
449 		(void) pthread_mutex_unlock(&session_p->session_mutex);
450 		return (CKR_OPERATION_ACTIVE);
451 	}
452 
453 	session_p->digest.flags = CRYPTO_OPERATION_ACTIVE;
454 
455 	(void) pthread_mutex_unlock(&session_p->session_mutex);
456 
457 	rv = soft_digest_init(session_p, pMechanism);
458 
459 	if (rv != CKR_OK) {
460 		(void) pthread_mutex_lock(&session_p->session_mutex);
461 		session_p->digest.flags &= ~CRYPTO_OPERATION_ACTIVE;
462 		(void) pthread_mutex_unlock(&session_p->session_mutex);
463 	}
464 
465 	return (rv);
466 }
467 
468 /*
469  * Call soft_digest_update() function with the value of a secret key.
470  */
471 CK_RV
472 soft_digest_key(soft_session_t *session_p, soft_object_t *key_p)
473 {
474 
475 	CK_RV rv;
476 
477 	/* Only secret key is allowed to be digested */
478 	if (key_p->class != CKO_SECRET_KEY)
479 		return (CKR_KEY_INDIGESTIBLE);
480 
481 	if ((OBJ_SEC_VALUE(key_p) == NULL) ||
482 	    (OBJ_SEC_VALUE_LEN(key_p) == 0))
483 		return (CKR_KEY_SIZE_RANGE);
484 
485 	rv = soft_digest_update(session_p, OBJ_SEC_VALUE(key_p),
486 	    OBJ_SEC_VALUE_LEN(key_p));
487 
488 	return (rv);
489 
490 }
491