xref: /illumos-gate/usr/src/lib/pkcs11/pkcs11_softtoken/common/softDigestUtil.c (revision eb9a1df2aeb866bf1de4494433b6d7e5fa07b3ae)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  * Copyright (c) 2018, Joyent, Inc.
25  */
26 
27 #include <strings.h>
28 #include <md5.h>
29 #include <pthread.h>
30 #include <stdlib.h>
31 #include <sys/sha1.h>
32 #include <sys/sha2.h>
33 #include <sys/types.h>
34 #include <security/cryptoki.h>
35 #include "softGlobal.h"
36 #include "softOps.h"
37 #include "softSession.h"
38 #include "softObject.h"
39 
40 
41 /*
42  * soft_digest_init()
43  *
44  * Arguments:
45  *	session_p:	pointer to soft_session_t struct
46  *	pMechanism:	pointer to CK_MECHANISM struct provided by application
47  *
48  * Description:
49  *	called by C_DigestInit(). This function allocates space for
50  *	context, then calls the corresponding software provided digest
51  *	init routine based on the mechanism.
52  *
53  * Returns:
54  *	CKR_OK: success
55  *	CKR_HOST_MEMORY: run out of system memory
56  *	CKR_MECHANISM_INVALID: invalid mechanism type
57  */
58 CK_RV
59 soft_digest_init(soft_session_t *session_p, CK_MECHANISM_PTR pMechanism)
60 {
61 
62 	switch (pMechanism->mechanism) {
63 
64 	case CKM_MD5:
65 		(void) pthread_mutex_lock(&session_p->session_mutex);
66 
67 		session_p->digest.context = malloc(sizeof (MD5_CTX));
68 
69 		if (session_p->digest.context == NULL) {
70 			(void) pthread_mutex_unlock(&session_p->session_mutex);
71 			return (CKR_HOST_MEMORY);
72 		}
73 
74 		session_p->digest.mech.mechanism = CKM_MD5;
75 		(void) pthread_mutex_unlock(&session_p->session_mutex);
76 
77 		MD5Init((MD5_CTX *)session_p->digest.context);
78 
79 		break;
80 
81 	case CKM_SHA_1:
82 
83 		(void) pthread_mutex_lock(&session_p->session_mutex);
84 
85 		session_p->digest.context = malloc(sizeof (SHA1_CTX));
86 
87 		if (session_p->digest.context == NULL) {
88 			(void) pthread_mutex_unlock(&session_p->session_mutex);
89 			return (CKR_HOST_MEMORY);
90 		}
91 
92 		session_p->digest.mech.mechanism = CKM_SHA_1;
93 		session_p->digest.mech.pParameter = pMechanism->pParameter;
94 		session_p->digest.mech.ulParameterLen =
95 		    pMechanism->ulParameterLen;
96 		(void) pthread_mutex_unlock(&session_p->session_mutex);
97 
98 		SHA1Init((SHA1_CTX *)session_p->digest.context);
99 
100 		break;
101 
102 	case CKM_SHA256:
103 	case CKM_SHA384:
104 	case CKM_SHA512:
105 	case CKM_SHA512_224:
106 	case CKM_SHA512_256:
107 
108 		(void) pthread_mutex_lock(&session_p->session_mutex);
109 
110 		session_p->digest.context = malloc(sizeof (SHA2_CTX));
111 
112 		if (session_p->digest.context == NULL) {
113 			(void) pthread_mutex_unlock(&session_p->session_mutex);
114 			return (CKR_HOST_MEMORY);
115 		}
116 
117 		session_p->digest.mech.mechanism = pMechanism->mechanism;
118 		(void) pthread_mutex_unlock(&session_p->session_mutex);
119 
120 		switch (pMechanism->mechanism) {
121 		case CKM_SHA256:
122 			SHA2Init(SHA256,
123 			    (SHA2_CTX *)session_p->digest.context);
124 			break;
125 
126 		case CKM_SHA384:
127 			SHA2Init(SHA384,
128 			    (SHA2_CTX *)session_p->digest.context);
129 			break;
130 
131 		case CKM_SHA512:
132 			SHA2Init(SHA512,
133 			    (SHA2_CTX *)session_p->digest.context);
134 			break;
135 		case CKM_SHA512_224:
136 			SHA2Init(SHA512_224,
137 			    (SHA2_CTX *)session_p->digest.context);
138 			break;
139 		case CKM_SHA512_256:
140 			SHA2Init(SHA512_256,
141 			    (SHA2_CTX *)session_p->digest.context);
142 			break;
143 		}
144 		break;
145 
146 	default:
147 		return (CKR_MECHANISM_INVALID);
148 	}
149 
150 	return (CKR_OK);
151 }
152 
153 
154 /*
155  * soft_digest_common()
156  *
157  * Arguments:
158  *      session_p:	pointer to soft_session_t struct
159  *	pData:		pointer to the input data to be digested
160  *	ulDataLen:	length of the input data
161  *	pDigest:	pointer to the output data after digesting
162  *	pulDigestLen:	length of the output data
163  *
164  * Description:
165  *      called by soft_digest() or soft_digest_final(). This function
166  *      determines the length of output buffer and calls the corresponding
167  *	software provided digest routine based on the mechanism.
168  *
169  * Returns:
170  *      CKR_OK: success
171  *      CKR_MECHANISM_INVALID: invalid mechanism type
172  *      CKR_BUFFER_TOO_SMALL: the output buffer provided by application
173  *			      is too small
174  */
175 CK_RV
176 soft_digest_common(soft_session_t *session_p, CK_BYTE_PTR pData,
177     CK_ULONG ulDataLen, CK_BYTE_PTR pDigest, CK_ULONG_PTR pulDigestLen)
178 {
179 
180 	CK_ULONG digestLen = 0;
181 	size_t len = 0;
182 
183 	/*
184 	 * Determine the output data length based on the mechanism
185 	 */
186 	switch (session_p->digest.mech.mechanism) {
187 
188 	case CKM_MD5:
189 		digestLen = 16;
190 		break;
191 
192 	case CKM_SHA_1:
193 		digestLen = SHA1_DIGEST_LENGTH;
194 		break;
195 
196 	case CKM_SHA256:
197 		digestLen = SHA256_DIGEST_LENGTH;
198 		break;
199 
200 	case CKM_SHA384:
201 		digestLen = SHA384_DIGEST_LENGTH;
202 		break;
203 
204 	case CKM_SHA512:
205 		digestLen = SHA512_DIGEST_LENGTH;
206 		break;
207 
208 	case CKM_SHA512_224:
209 		digestLen = SHA512_224_DIGEST_LENGTH;
210 		break;
211 
212 	case CKM_SHA512_256:
213 		digestLen = SHA512_256_DIGEST_LENGTH;
214 		break;
215 
216 	default:
217 		return (CKR_MECHANISM_INVALID);
218 	}
219 
220 	if (pDigest == NULL) {
221 		/*
222 		 * Application only wants to know the length of the
223 		 * buffer needed to hold the message digest.
224 		 */
225 		*pulDigestLen = digestLen;
226 		return (CKR_OK);
227 	}
228 
229 	if (*pulDigestLen < digestLen) {
230 		/*
231 		 * Application provides buffer too small to hold the
232 		 * digest message. Return the length of buffer needed
233 		 * to the application.
234 		 */
235 		*pulDigestLen = digestLen;
236 		return (CKR_BUFFER_TOO_SMALL);
237 	}
238 
239 	/*
240 	 * Call the corresponding system provided software digest routine.
241 	 * If the soft_digest_common() is called by soft_digest_final()
242 	 * the pData is NULL, and the ulDataLen is zero.
243 	 */
244 	switch (session_p->digest.mech.mechanism) {
245 
246 	case CKM_MD5:
247 		if (pData != NULL) {
248 			/*
249 			 * this is called by soft_digest()
250 			 */
251 #ifdef	__sparcv9
252 			MD5Update((MD5_CTX *)session_p->digest.context,
253 			    /* LINTED */
254 			    pData, (uint_t)ulDataLen);
255 #else	/* !__sparcv9 */
256 			MD5Update((MD5_CTX *)session_p->digest.context,
257 			    pData, ulDataLen);
258 #endif	/* __sparcv9 */
259 			MD5Final(pDigest, (MD5_CTX *)session_p->digest.context);
260 		} else {
261 			/*
262 			 * this is called by soft_digest_final()
263 			 */
264 			MD5Final(pDigest, (MD5_CTX *)session_p->digest.context);
265 			len = sizeof (MD5_CTX);
266 		}
267 		break;
268 
269 	case CKM_SHA_1:
270 		if (pData != NULL) {
271 			/*
272 			 * this is called by soft_digest()
273 			 */
274 
275 #ifdef	__sparcv9
276 			SHA1Update((SHA1_CTX *)session_p->digest.context,
277 			    /* LINTED */
278 			    pData, (uint32_t)ulDataLen);
279 #else	/* !__sparcv9 */
280 			SHA1Update((SHA1_CTX *)session_p->digest.context,
281 			    pData, ulDataLen);
282 #endif	/* __sparcv9 */
283 			SHA1Final(pDigest,
284 			    (SHA1_CTX *)session_p->digest.context);
285 		} else {
286 			/*
287 			 * this is called by soft_digest_final()
288 			 */
289 			SHA1Final(pDigest,
290 			    (SHA1_CTX *)session_p->digest.context);
291 			len = sizeof (SHA1_CTX);
292 		}
293 		break;
294 	case CKM_SHA256:
295 	case CKM_SHA384:
296 	case CKM_SHA512:
297 	case CKM_SHA512_224:
298 	case CKM_SHA512_256:
299 		if (pData != NULL) {
300 			/*
301 			 * this is called by soft_digest()
302 			 */
303 
304 			SHA2Update((SHA2_CTX *)session_p->digest.context,
305 			    pData, ulDataLen);
306 
307 			SHA2Final(pDigest,
308 			    (SHA2_CTX *)session_p->digest.context);
309 		} else {
310 			/*
311 			 * this is called by soft_digest_final()
312 			 */
313 			SHA2Final(pDigest,
314 			    (SHA2_CTX *)session_p->digest.context);
315 			len = sizeof (SHA2_CTX);
316 		}
317 
318 		break;
319 	}
320 
321 	/* Paranoia on behalf of C_DigestKey callers: bzero the context */
322 	if (session_p->digest.flags & CRYPTO_KEY_DIGESTED) {
323 		explicit_bzero(session_p->digest.context, len);
324 		session_p->digest.flags &= ~CRYPTO_KEY_DIGESTED;
325 	}
326 	*pulDigestLen = digestLen;
327 	(void) pthread_mutex_lock(&session_p->session_mutex);
328 	free(session_p->digest.context);
329 	session_p->digest.context = NULL;
330 	(void) pthread_mutex_unlock(&session_p->session_mutex);
331 
332 	return (CKR_OK);
333 }
334 
335 
336 /*
337  * soft_digest()
338  *
339  * Arguments:
340  *      session_p:	pointer to soft_session_t struct
341  *      pData:		pointer to the input data to be digested
342  *      ulDataLen:	length of the input data
343  *      pDigest:	pointer to the output data after digesting
344  *      pulDigestLen:	length of the output data
345  *
346  * Description:
347  *      called by C_Digest(). This function calls soft_digest_common().
348  *
349  * Returns:
350  *      see return values in soft_digest_common().
351  */
352 CK_RV
353 soft_digest(soft_session_t *session_p, CK_BYTE_PTR pData, CK_ULONG ulDataLen,
354     CK_BYTE_PTR pDigest, CK_ULONG_PTR pulDigestLen)
355 {
356 
357 	return (soft_digest_common(session_p, pData, ulDataLen,
358 	    pDigest, pulDigestLen));
359 }
360 
361 
362 /*
363  * soft_digest_update()
364  *
365  * Arguments:
366  *      session_p:	pointer to soft_session_t struct
367  *      pPart:		pointer to the input data to be digested
368  *      ulPartLen:	length of the input data
369  *
370  * Description:
371  *      called by C_DigestUpdate(). This function calls the corresponding
372  *	software provided digest update routine based on the mechanism.
373  *
374  * Returns:
375  *      CKR_OK: success
376  *      CKR_MECHANISM_INVALID: invalid MECHANISM type.
377  */
378 CK_RV
379 soft_digest_update(soft_session_t *session_p, CK_BYTE_PTR pPart,
380     CK_ULONG ulPartLen)
381 {
382 
383 	switch (session_p->digest.mech.mechanism) {
384 
385 	case CKM_MD5:
386 #ifdef	__sparcv9
387 		MD5Update((MD5_CTX *)session_p->digest.context,
388 		    /* LINTED */
389 		    pPart, (uint_t)ulPartLen);
390 #else	/* !__sparcv9 */
391 		MD5Update((MD5_CTX *)session_p->digest.context,
392 		    pPart, ulPartLen);
393 #endif	/* __sparcv9 */
394 		break;
395 
396 	case CKM_SHA_1:
397 #ifdef	__sparcv9
398 		SHA1Update((SHA1_CTX *)session_p->digest.context,
399 		    /* LINTED */
400 		    pPart, (uint32_t)ulPartLen);
401 #else	/* !__sparcv9 */
402 		SHA1Update((SHA1_CTX *)session_p->digest.context,
403 		    pPart, ulPartLen);
404 #endif	/* __sparcv9 */
405 		break;
406 
407 	case CKM_SHA256:
408 	case CKM_SHA384:
409 	case CKM_SHA512:
410 	case CKM_SHA512_224:
411 	case CKM_SHA512_256:
412 		SHA2Update((SHA2_CTX *)session_p->digest.context,
413 		    pPart, ulPartLen);
414 		break;
415 
416 	default:
417 		return (CKR_MECHANISM_INVALID);
418 	}
419 
420 	return (CKR_OK);
421 }
422 
423 
424 /*
425  * soft_digest_final()
426  *
427  * Arguments:
428  *      session_p:	pointer to soft_session_t struct
429  *      pDigest:	pointer to the output data after digesting
430  *      pulDigestLen:	length of the output data
431  *
432  * Description:
433  *      called by C_DigestFinal(). This function calls soft_digest_common().
434  *
435  * Returns:
436  *	see return values in soft_digest_common().
437  */
438 CK_RV
439 soft_digest_final(soft_session_t *session_p, CK_BYTE_PTR pDigest,
440     CK_ULONG_PTR pulDigestLen)
441 {
442 
443 	return (soft_digest_common(session_p, NULL, 0,
444 	    pDigest, pulDigestLen));
445 }
446 
447 /*
448  * Perform digest init operation internally for the support of
449  * CKM_MD5_RSA_PKCS, CKM_SHA1_RSA_PKCS, CKM_SHA1_KEY_DERIVATION
450  * and CKM_MD5_KEY_DERIVATION mechanisms.
451  *
452  * This function is called with the session being held, and without
453  * its mutex taken.
454  */
455 CK_RV
456 soft_digest_init_internal(soft_session_t *session_p,
457     CK_MECHANISM_PTR pMechanism)
458 {
459 
460 	CK_RV rv;
461 
462 	(void) pthread_mutex_lock(&session_p->session_mutex);
463 
464 	/* Check to see if digest operation is already active */
465 	if (session_p->digest.flags & CRYPTO_OPERATION_ACTIVE) {
466 		(void) pthread_mutex_unlock(&session_p->session_mutex);
467 		return (CKR_OPERATION_ACTIVE);
468 	}
469 
470 	session_p->digest.flags = CRYPTO_OPERATION_ACTIVE;
471 
472 	(void) pthread_mutex_unlock(&session_p->session_mutex);
473 
474 	rv = soft_digest_init(session_p, pMechanism);
475 
476 	if (rv != CKR_OK) {
477 		(void) pthread_mutex_lock(&session_p->session_mutex);
478 		session_p->digest.flags &= ~CRYPTO_OPERATION_ACTIVE;
479 		(void) pthread_mutex_unlock(&session_p->session_mutex);
480 	}
481 
482 	return (rv);
483 }
484 
485 /*
486  * Call soft_digest_update() function with the value of a secret key.
487  */
488 CK_RV
489 soft_digest_key(soft_session_t *session_p, soft_object_t *key_p)
490 {
491 
492 	CK_RV rv;
493 
494 	/* Only secret key is allowed to be digested */
495 	if (key_p->class != CKO_SECRET_KEY)
496 		return (CKR_KEY_INDIGESTIBLE);
497 
498 	if ((OBJ_SEC_VALUE(key_p) == NULL) ||
499 	    (OBJ_SEC_VALUE_LEN(key_p) == 0))
500 		return (CKR_KEY_SIZE_RANGE);
501 
502 	rv = soft_digest_update(session_p, OBJ_SEC_VALUE(key_p),
503 	    OBJ_SEC_VALUE_LEN(key_p));
504 
505 	return (rv);
506 
507 }
508 
509 /*
510  * This function releases allocated digest context. The caller
511  * may (lock_held == B_TRUE) or may not (lock_held == B_FALSE)
512  * hold a session mutex.
513  */
514 void
515 soft_digest_cleanup(soft_session_t *session_p, boolean_t lock_held)
516 {
517 	boolean_t lock_true = B_TRUE;
518 
519 	if (!lock_held)
520 		(void) pthread_mutex_lock(&session_p->session_mutex);
521 
522 	if (session_p->digest.context != NULL) {
523 		free(session_p->digest.context);
524 		session_p->digest.context = NULL;
525 	}
526 
527 	session_p->digest.flags = 0;
528 
529 	if (!lock_held)
530 		SES_REFRELE(session_p, lock_true);
531 
532 }
533