xref: /titanic_51/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelDecrypt.c (revision c5aee8047f101b999b8f96ced79e802071372a52)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <pthread.h>
27 #include <stdlib.h>
28 #include <errno.h>
29 #include <sys/crypto/ioctl.h>
30 #include <security/cryptoki.h>
31 #include "kernelGlobal.h"
32 #include "kernelSession.h"
33 #include "kernelObject.h"
34 
35 
36 /*
37  * Real decryptInit work. The caller doesn't hold the session lock.
38  */
39 CK_RV
40 kernel_decrypt_init(kernel_session_t *session_p, kernel_object_t *key_p,
41     CK_MECHANISM_PTR pMechanism)
42 {
43 	CK_RV rv;
44 	crypto_decrypt_init_t decrypt_init;
45 	crypto_mech_type_t k_mech_type;
46 	boolean_t ses_lock_held = B_FALSE;
47 	int r;
48 
49 	/* Check to see if key object allows for decryption. */
50 	if (key_p->is_lib_obj && !(key_p->bool_attr_mask & DECRYPT_BOOL_ON)) {
51 		return (CKR_KEY_TYPE_INCONSISTENT);
52 	}
53 
54 	/* Get the kernel's internal mechanism number. */
55 	rv = kernel_mech(pMechanism->mechanism, &k_mech_type);
56 	if (rv != CKR_OK)
57 		return (rv);
58 
59 	(void) pthread_mutex_lock(&session_p->session_mutex);
60 	ses_lock_held = B_TRUE;
61 
62 	/*
63 	 * This active flag will remain ON until application calls either
64 	 * C_Decrypt or C_DecryptFinal to actually obtain the final piece
65 	 * of plaintext.
66 	 */
67 	session_p->decrypt.flags = CRYPTO_OPERATION_ACTIVE;
68 
69 	/* set up key data */
70 	if (!key_p->is_lib_obj) {
71 		decrypt_init.di_key.ck_format = CRYPTO_KEY_REFERENCE;
72 		decrypt_init.di_key.ck_obj_id = key_p->k_handle;
73 	} else {
74 		if (key_p->class == CKO_SECRET_KEY) {
75 			decrypt_init.di_key.ck_format = CRYPTO_KEY_RAW;
76 			decrypt_init.di_key.ck_data =
77 			    get_symmetric_key_value(key_p);
78 			if (decrypt_init.di_key.ck_data == NULL) {
79 				rv = CKR_HOST_MEMORY;
80 				goto clean_exit;
81 			}
82 			/* KEF key lengths are expressed in bits */
83 			decrypt_init.di_key.ck_length =
84 			    OBJ_SEC(key_p)->sk_value_len << 3;
85 
86 		} else if (key_p->key_type == CKK_RSA) {
87 			if (get_rsa_private_key(key_p, &decrypt_init.di_key) !=
88 			    CKR_OK) {
89 				rv = CKR_HOST_MEMORY;
90 				goto clean_exit;
91 			}
92 		} else {
93 			rv = CKR_KEY_TYPE_INCONSISTENT;
94 			goto clean_exit;
95 		}
96 	}
97 
98 	decrypt_init.di_session = session_p->k_session;
99 	session_p->decrypt.mech = *pMechanism;
100 
101 	/* Cache this capability value for efficiency */
102 	if (INPLACE_MECHANISM(session_p->decrypt.mech.mechanism)) {
103 		session_p->decrypt.flags |= CRYPTO_OPERATION_INPLACE_OK;
104 	}
105 	(void) pthread_mutex_unlock(&session_p->session_mutex);
106 
107 	ses_lock_held = B_FALSE;
108 	decrypt_init.di_mech.cm_type = k_mech_type;
109 	decrypt_init.di_mech.cm_param = pMechanism->pParameter;
110 	decrypt_init.di_mech.cm_param_len = pMechanism->ulParameterLen;
111 
112 	while ((r = ioctl(kernel_fd, CRYPTO_DECRYPT_INIT, &decrypt_init)) < 0) {
113 		if (errno != EINTR)
114 			break;
115 	}
116 	if (r < 0) {
117 		rv = CKR_FUNCTION_FAILED;
118 	} else {
119 		rv = crypto2pkcs11_error_number(decrypt_init.di_return_value);
120 	}
121 
122 	/* Free memory allocated for decrypt_init.di_key */
123 	if (key_p->is_lib_obj) {
124 		if (key_p->class == CKO_SECRET_KEY) {
125 			free(decrypt_init.di_key.ck_data);
126 		} else if (key_p->key_type == CKK_RSA) {
127 			free_key_attributes(&decrypt_init.di_key);
128 		}
129 	}
130 
131 clean_exit:
132 
133 	if (!ses_lock_held) {
134 		(void) pthread_mutex_lock(&session_p->session_mutex);
135 		ses_lock_held = B_TRUE;
136 	}
137 
138 	if (rv != CKR_OK)
139 		session_p->decrypt.flags &= ~CRYPTO_OPERATION_ACTIVE;
140 
141 	if (ses_lock_held) {
142 		(void) pthread_mutex_unlock(&session_p->session_mutex);
143 		ses_lock_held = B_FALSE;
144 	}
145 
146 	return (rv);
147 }
148 
149 CK_RV
150 C_DecryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism,
151     CK_OBJECT_HANDLE hKey)
152 {
153 
154 	CK_RV rv;
155 	kernel_session_t *session_p;
156 	kernel_object_t	*key_p;
157 	boolean_t ses_lock_held = B_FALSE;
158 
159 	if (!kernel_initialized)
160 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
161 
162 	if (pMechanism == NULL) {
163 		return (CKR_ARGUMENTS_BAD);
164 	}
165 
166 	/* Obtain the session pointer. */
167 	rv = handle2session(hSession, &session_p);
168 	if (rv != CKR_OK)
169 		return (rv);
170 
171 	/* Obtain the object pointer. */
172 	HANDLE2OBJECT(hKey, key_p, rv);
173 	if (rv == CKR_OK) {
174 		rv = kernel_decrypt_init(session_p, key_p, pMechanism);
175 		OBJ_REFRELE(key_p);
176 	}
177 
178 	REFRELE(session_p, ses_lock_held);
179 	return (rv);
180 }
181 
182 
183 
184 /*
185  * Real decrypt work. The caller doesn't hold the session lock.
186  */
187 CK_RV
188 kernel_decrypt(kernel_session_t *session_p, CK_BYTE_PTR pEncryptedData,
189     CK_ULONG ulEncryptedData, CK_BYTE_PTR pData, CK_ULONG_PTR pulDataLen)
190 {
191 	crypto_decrypt_t decrypt;
192 	boolean_t ses_lock_held = B_FALSE;
193 	boolean_t inplace;
194 	CK_RV rv;
195 	int r;
196 
197 	(void) pthread_mutex_lock(&session_p->session_mutex);
198 	ses_lock_held = B_TRUE;
199 
200 	/* Application must call C_DecryptInit before calling C_Decrypt. */
201 	if (!(session_p->decrypt.flags & CRYPTO_OPERATION_ACTIVE)) {
202 		rv = CKR_OPERATION_NOT_INITIALIZED;
203 		goto clean_exit;
204 	}
205 
206 	/*
207 	 * C_Decrypt must be called without intervening C_DecryptUpdate
208 	 * calls.
209 	 */
210 	if (session_p->decrypt.flags & CRYPTO_OPERATION_UPDATE) {
211 		/*
212 		 * C_Decrypt cannot be used to terminate a multiple-part
213 		 * operation, so we'll leave the active decrypt operation
214 		 * flag on and let the application continue with the
215 		 * decrypt update operation.
216 		 */
217 		rv = CKR_FUNCTION_FAILED;
218 		goto clean_exit;
219 	}
220 
221 	decrypt.cd_session = session_p->k_session;
222 
223 	/*
224 	 * Certain mechanisms, where the length of the plaintext is
225 	 * same as the transformed ciphertext, can be optimized
226 	 * by the kernel into an in-place operation. Unfortunately,
227 	 * some applications use a plaintext buffer that is larger
228 	 * than it needs to be. We fix that here.
229 	 */
230 	inplace = (session_p->decrypt.flags & CRYPTO_OPERATION_INPLACE_OK) != 0;
231 
232 	if (ulEncryptedData < *pulDataLen && inplace) {
233 		decrypt.cd_datalen = ulEncryptedData;
234 	} else {
235 		decrypt.cd_datalen = *pulDataLen;
236 	}
237 	(void) pthread_mutex_unlock(&session_p->session_mutex);
238 	ses_lock_held = B_FALSE;
239 
240 	decrypt.cd_databuf = (char *)pData;
241 	decrypt.cd_encrlen = ulEncryptedData;
242 	decrypt.cd_encrbuf = (char *)pEncryptedData;
243 	decrypt.cd_flags =
244 	    ((inplace && (pData != NULL)) || (pData == pEncryptedData)) &&
245 	    (decrypt.cd_datalen == decrypt.cd_encrlen) ?
246 	    CRYPTO_INPLACE_OPERATION : 0;
247 
248 	while ((r = ioctl(kernel_fd, CRYPTO_DECRYPT, &decrypt)) < 0) {
249 		if (errno != EINTR)
250 			break;
251 	}
252 	if (r < 0) {
253 		rv = CKR_FUNCTION_FAILED;
254 	} else {
255 		rv = crypto2pkcs11_error_number(decrypt.cd_return_value);
256 	}
257 
258 	if (rv == CKR_OK || rv == CKR_BUFFER_TOO_SMALL)
259 		*pulDataLen = decrypt.cd_datalen;
260 
261 clean_exit:
262 
263 	if (ses_lock_held)
264 		(void) pthread_mutex_unlock(&session_p->session_mutex);
265 
266 	return (rv);
267 }
268 
269 CK_RV
270 C_Decrypt(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pEncryptedData,
271     CK_ULONG ulEncryptedData, CK_BYTE_PTR pData, CK_ULONG_PTR pulDataLen)
272 {
273 	CK_RV rv;
274 	kernel_session_t *session_p;
275 	boolean_t ses_lock_held = B_FALSE;
276 
277 	if (!kernel_initialized)
278 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
279 
280 	/* Obtain the session pointer. */
281 	rv = handle2session(hSession, &session_p);
282 	if (rv != CKR_OK)
283 		return (rv);
284 
285 	/*
286 	 * No need to check pData because application might
287 	 * just want to know the length of decrypted data.
288 	 */
289 	if (pulDataLen == NULL) {
290 		rv = CKR_ARGUMENTS_BAD;
291 		goto clean_exit;
292 	}
293 
294 	rv = kernel_decrypt(session_p, pEncryptedData, ulEncryptedData, pData,
295 	    pulDataLen);
296 
297 	if ((rv == CKR_BUFFER_TOO_SMALL) ||
298 	    (rv == CKR_OK && pData == NULL)) {
299 		/*
300 		 * We will not terminate the active decrypt operation flag,
301 		 * when the application-supplied buffer is too small, or
302 		 * the application asks for the length of buffer to hold
303 		 * the plaintext.
304 		 */
305 		REFRELE(session_p, ses_lock_held);
306 		return (rv);
307 	}
308 
309 clean_exit:
310 	/*
311 	 * Terminates the active decrypt operation.
312 	 * Application needs to call C_DecryptInit again for next
313 	 * decrypt operation.
314 	 */
315 	(void) pthread_mutex_lock(&session_p->session_mutex);
316 	session_p->decrypt.flags = 0;
317 	ses_lock_held = B_TRUE;
318 	REFRELE(session_p, ses_lock_held);
319 
320 	return (rv);
321 }
322 
323 
324 CK_RV
325 C_DecryptUpdate(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pEncryptedPart,
326     CK_ULONG ulEncryptedPartLen, CK_BYTE_PTR pPart,
327     CK_ULONG_PTR pulPartLen)
328 {
329 
330 	CK_RV rv;
331 	kernel_session_t *session_p;
332 	boolean_t ses_lock_held = B_FALSE;
333 	boolean_t inplace;
334 	crypto_decrypt_update_t decrypt_update;
335 	int r;
336 
337 	if (!kernel_initialized)
338 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
339 
340 	/* Obtain the session pointer. */
341 	rv = handle2session(hSession, &session_p);
342 	if (rv != CKR_OK)
343 		return (rv);
344 
345 	if (pEncryptedPart == NULL) {
346 		rv = CKR_ARGUMENTS_BAD;
347 		goto clean_exit;
348 	}
349 
350 	/*
351 	 * Only check if pulPartLen is NULL.
352 	 * No need to check if pPart is NULL because application
353 	 * might just ask for the length of buffer to hold the
354 	 * recovered data.
355 	 */
356 	if (pulPartLen == NULL) {
357 		rv = CKR_ARGUMENTS_BAD;
358 		goto clean_exit;
359 	}
360 
361 	(void) pthread_mutex_lock(&session_p->session_mutex);
362 	ses_lock_held = B_TRUE;
363 
364 	/*
365 	 * Application must call C_DecryptInit before calling
366 	 * C_DecryptUpdate.
367 	 */
368 	if (!(session_p->decrypt.flags & CRYPTO_OPERATION_ACTIVE)) {
369 		REFRELE(session_p, ses_lock_held);
370 		return (CKR_OPERATION_NOT_INITIALIZED);
371 	}
372 
373 	session_p->decrypt.flags |= CRYPTO_OPERATION_UPDATE;
374 
375 	decrypt_update.du_session = session_p->k_session;
376 	(void) pthread_mutex_unlock(&session_p->session_mutex);
377 	ses_lock_held = B_FALSE;
378 
379 	decrypt_update.du_datalen = *pulPartLen;
380 	decrypt_update.du_databuf = (char *)pPart;
381 	decrypt_update.du_encrlen = ulEncryptedPartLen;
382 	decrypt_update.du_encrbuf = (char *)pEncryptedPart;
383 
384 	inplace = (session_p->decrypt.flags & CRYPTO_OPERATION_INPLACE_OK) != 0;
385 	decrypt_update.du_flags =
386 	    ((inplace && (pPart != NULL)) || (pPart == pEncryptedPart)) &&
387 	    (decrypt_update.du_datalen == decrypt_update.du_encrlen) ?
388 	    CRYPTO_INPLACE_OPERATION : 0;
389 
390 	while ((r = ioctl(kernel_fd, CRYPTO_DECRYPT_UPDATE,
391 	    &decrypt_update)) < 0) {
392 		if (errno != EINTR)
393 			break;
394 	}
395 	if (r < 0) {
396 		rv = CKR_FUNCTION_FAILED;
397 	} else {
398 		rv = crypto2pkcs11_error_number(
399 		    decrypt_update.du_return_value);
400 	}
401 
402 	/*
403 	 * If CKR_OK or CKR_BUFFER_TOO_SMALL, set the output length.
404 	 * We don't terminate the current decryption operation.
405 	 */
406 	if (rv == CKR_OK || rv == CKR_BUFFER_TOO_SMALL) {
407 		*pulPartLen = decrypt_update.du_datalen;
408 		REFRELE(session_p, ses_lock_held);
409 		return (rv);
410 	}
411 
412 clean_exit:
413 	/*
414 	 * After an error occurred, terminate the current decrypt
415 	 * operation by resetting the active and update flags.
416 	 */
417 	(void) pthread_mutex_lock(&session_p->session_mutex);
418 	session_p->decrypt.flags = 0;
419 	ses_lock_held = B_TRUE;
420 	REFRELE(session_p, ses_lock_held);
421 
422 	return (rv);
423 }
424 
425 
426 CK_RV
427 C_DecryptFinal(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pLastPart,
428     CK_ULONG_PTR pulLastPartLen)
429 {
430 
431 	CK_RV rv;
432 	kernel_session_t *session_p;
433 	boolean_t ses_lock_held = B_FALSE;
434 	crypto_decrypt_final_t decrypt_final;
435 	int r;
436 
437 	if (!kernel_initialized)
438 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
439 
440 	/* Obtain the session pointer. */
441 	rv = handle2session(hSession, &session_p);
442 	if (rv != CKR_OK)
443 		return (rv);
444 
445 	if (pulLastPartLen == NULL) {
446 		rv = CKR_ARGUMENTS_BAD;
447 		goto clean_exit;
448 	}
449 
450 	(void) pthread_mutex_lock(&session_p->session_mutex);
451 	ses_lock_held = B_TRUE;
452 
453 	/*
454 	 * Application must call C_DecryptInit before calling
455 	 * C_DecryptFinal.
456 	 */
457 	if (!(session_p->decrypt.flags & CRYPTO_OPERATION_ACTIVE)) {
458 		REFRELE(session_p, ses_lock_held);
459 		return (CKR_OPERATION_NOT_INITIALIZED);
460 	}
461 
462 	decrypt_final.df_session = session_p->k_session;
463 	(void) pthread_mutex_unlock(&session_p->session_mutex);
464 	ses_lock_held = B_FALSE;
465 
466 	decrypt_final.df_datalen = *pulLastPartLen;
467 	decrypt_final.df_databuf = (char *)pLastPart;
468 
469 	while ((r = ioctl(kernel_fd, CRYPTO_DECRYPT_FINAL,
470 	    &decrypt_final)) < 0) {
471 		if (errno != EINTR)
472 			break;
473 	}
474 	if (r < 0) {
475 		rv = CKR_FUNCTION_FAILED;
476 	} else {
477 		rv = crypto2pkcs11_error_number(decrypt_final.df_return_value);
478 	}
479 
480 	if (rv == CKR_OK || rv == CKR_BUFFER_TOO_SMALL)
481 		*pulLastPartLen = decrypt_final.df_datalen;
482 
483 	if (rv == CKR_BUFFER_TOO_SMALL ||
484 	    (rv == CKR_OK && pLastPart == NULL)) {
485 		/*
486 		 * We will not terminate the active decrypt operation flag,
487 		 * when the application-supplied buffer is too small, or
488 		 * the application asks for the length of buffer to hold
489 		 * the plaintext.
490 		 */
491 		REFRELE(session_p, ses_lock_held);
492 		return (rv);
493 	}
494 
495 clean_exit:
496 	/* Terminates the active decrypt operation */
497 	(void) pthread_mutex_lock(&session_p->session_mutex);
498 	session_p->decrypt.flags = 0;
499 	ses_lock_held = B_TRUE;
500 	REFRELE(session_p, ses_lock_held);
501 
502 	return (rv);
503 }
504