xref: /illumos-gate/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelDecrypt.c (revision d321a33cdd896e6b211d113a33698dd76e89b861)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <pthread.h>
29 #include <stdlib.h>
30 #include <errno.h>
31 #include <sys/crypto/ioctl.h>
32 #include <security/cryptoki.h>
33 #include "kernelGlobal.h"
34 #include "kernelSession.h"
35 #include "kernelObject.h"
36 
37 
38 /*
39  * Real decryptInit work. The caller doesn't hold the session lock.
40  */
41 CK_RV
42 kernel_decrypt_init(kernel_session_t *session_p, kernel_object_t *key_p,
43     CK_MECHANISM_PTR pMechanism)
44 {
45 	CK_RV rv;
46 	crypto_decrypt_init_t decrypt_init;
47 	crypto_mech_type_t k_mech_type;
48 	boolean_t ses_lock_held = B_FALSE;
49 	int r;
50 
51 	/* Check to see if key object allows for decryption. */
52 	if (key_p->is_lib_obj && !(key_p->bool_attr_mask & DECRYPT_BOOL_ON)) {
53 		return (CKR_KEY_TYPE_INCONSISTENT);
54 	}
55 
56 	/* Get the kernel's internal mechanism number. */
57 	rv = kernel_mech(pMechanism->mechanism, &k_mech_type);
58 	if (rv != CKR_OK)
59 		return (rv);
60 
61 	(void) pthread_mutex_lock(&session_p->session_mutex);
62 	ses_lock_held = B_TRUE;
63 
64 	/*
65 	 * This active flag will remain ON until application calls either
66 	 * C_Decrypt or C_DecryptFinal to actually obtain the final piece
67 	 * of plaintext.
68 	 */
69 	session_p->decrypt.flags = CRYPTO_OPERATION_ACTIVE;
70 
71 	/* set up key data */
72 	if (!key_p->is_lib_obj) {
73 		decrypt_init.di_key.ck_format = CRYPTO_KEY_REFERENCE;
74 		decrypt_init.di_key.ck_obj_id = key_p->k_handle;
75 	} else {
76 		if (key_p->class == CKO_SECRET_KEY) {
77 			decrypt_init.di_key.ck_format = CRYPTO_KEY_RAW;
78 			decrypt_init.di_key.ck_data =
79 			    get_symmetric_key_value(key_p);
80 			if (decrypt_init.di_key.ck_data == NULL) {
81 				rv = CKR_HOST_MEMORY;
82 				goto clean_exit;
83 			}
84 			/* KEF key lengths are expressed in bits */
85 			decrypt_init.di_key.ck_length =
86 			    OBJ_SEC(key_p)->sk_value_len << 3;
87 
88 		} else if (key_p->key_type == CKK_RSA) {
89 			if (get_rsa_private_key(key_p, &decrypt_init.di_key) !=
90 			    CKR_OK) {
91 				rv = CKR_HOST_MEMORY;
92 				goto clean_exit;
93 			}
94 		} else {
95 			rv = CKR_KEY_TYPE_INCONSISTENT;
96 			goto clean_exit;
97 		}
98 	}
99 
100 	decrypt_init.di_session = session_p->k_session;
101 	session_p->decrypt.mech = *pMechanism;
102 	(void) pthread_mutex_unlock(&session_p->session_mutex);
103 	ses_lock_held = B_FALSE;
104 	decrypt_init.di_mech.cm_type = k_mech_type;
105 	decrypt_init.di_mech.cm_param = pMechanism->pParameter;
106 	decrypt_init.di_mech.cm_param_len = pMechanism->ulParameterLen;
107 
108 	while ((r = ioctl(kernel_fd, CRYPTO_DECRYPT_INIT, &decrypt_init)) < 0) {
109 		if (errno != EINTR)
110 			break;
111 	}
112 	if (r < 0) {
113 		rv = CKR_FUNCTION_FAILED;
114 	} else {
115 		rv = crypto2pkcs11_error_number(decrypt_init.di_return_value);
116 	}
117 
118 	/* Free memory allocated for decrypt_init.di_key */
119 	if (key_p->is_lib_obj) {
120 		if (key_p->class == CKO_SECRET_KEY) {
121 			free(decrypt_init.di_key.ck_data);
122 		} else if (key_p->key_type == CKK_RSA) {
123 			free_key_attributes(&decrypt_init.di_key);
124 		}
125 	}
126 
127 clean_exit:
128 
129 	if (!ses_lock_held) {
130 		(void) pthread_mutex_lock(&session_p->session_mutex);
131 		ses_lock_held = B_TRUE;
132 	}
133 
134 	if (rv != CKR_OK)
135 		session_p->decrypt.flags &= ~CRYPTO_OPERATION_ACTIVE;
136 
137 	if (ses_lock_held) {
138 		(void) pthread_mutex_unlock(&session_p->session_mutex);
139 		ses_lock_held = B_FALSE;
140 	}
141 
142 	return (rv);
143 }
144 
145 CK_RV
146 C_DecryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism,
147     CK_OBJECT_HANDLE hKey)
148 {
149 
150 	CK_RV rv;
151 	kernel_session_t *session_p;
152 	kernel_object_t	*key_p;
153 	boolean_t ses_lock_held = B_FALSE;
154 
155 	if (!kernel_initialized)
156 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
157 
158 	if (pMechanism == NULL) {
159 		return (CKR_ARGUMENTS_BAD);
160 	}
161 
162 	/* Obtain the session pointer. */
163 	rv = handle2session(hSession, &session_p);
164 	if (rv != CKR_OK)
165 		return (rv);
166 
167 	/* Obtain the object pointer. */
168 	HANDLE2OBJECT(hKey, key_p, rv);
169 	if (rv == CKR_OK) {
170 		rv = kernel_decrypt_init(session_p, key_p, pMechanism);
171 		OBJ_REFRELE(key_p);
172 	}
173 
174 	REFRELE(session_p, ses_lock_held);
175 	return (rv);
176 }
177 
178 
179 
180 /*
181  * Real decrypt work. The caller doesn't hold the session lock.
182  */
183 CK_RV
184 kernel_decrypt(kernel_session_t *session_p, CK_BYTE_PTR pEncryptedData,
185     CK_ULONG ulEncryptedData, CK_BYTE_PTR pData, CK_ULONG_PTR pulDataLen)
186 {
187 	crypto_decrypt_t decrypt;
188 	boolean_t ses_lock_held = B_FALSE;
189 	boolean_t inplace;
190 	CK_RV rv;
191 	int r;
192 
193 	(void) pthread_mutex_lock(&session_p->session_mutex);
194 	ses_lock_held = B_TRUE;
195 
196 	/* Application must call C_DecryptInit before calling C_Decrypt. */
197 	if (!(session_p->decrypt.flags & CRYPTO_OPERATION_ACTIVE)) {
198 		rv = CKR_OPERATION_NOT_INITIALIZED;
199 		goto clean_exit;
200 	}
201 
202 	/*
203 	 * C_Decrypt must be called without intervening C_DecryptUpdate
204 	 * calls.
205 	 */
206 	if (session_p->decrypt.flags & CRYPTO_OPERATION_UPDATE) {
207 		/*
208 		 * C_Decrypt cannot be used to terminate a multiple-part
209 		 * operation, so we'll leave the active decrypt operation
210 		 * flag on and let the application continue with the
211 		 * decrypt update operation.
212 		 */
213 		rv = CKR_FUNCTION_FAILED;
214 		goto clean_exit;
215 	}
216 
217 	decrypt.cd_session = session_p->k_session;
218 
219 	/*
220 	 * Certain mechanisms, where the length of the plaintext is
221 	 * same as the transformed ciphertext, can be optimized
222 	 * by the kernel into an in-place operation. Unfortunately,
223 	 * some applications use a plaintext buffer that is larger
224 	 * than it needs to be. We fix that here.
225 	 */
226 	inplace = INPLACE_MECHANISM(session_p->decrypt.mech.mechanism);
227 	if (ulEncryptedData < *pulDataLen && inplace) {
228 		decrypt.cd_datalen = ulEncryptedData;
229 	} else {
230 		decrypt.cd_datalen = *pulDataLen;
231 	}
232 	(void) pthread_mutex_unlock(&session_p->session_mutex);
233 	ses_lock_held = B_FALSE;
234 
235 	decrypt.cd_databuf = (char *)pData;
236 	decrypt.cd_encrlen = ulEncryptedData;
237 	decrypt.cd_encrbuf = (char *)pEncryptedData;
238 	decrypt.cd_flags = inplace && pData != NULL &&
239 	    decrypt.cd_datalen == decrypt.cd_encrlen ?
240 	    CRYPTO_INPLACE_OPERATION : 0;
241 
242 	while ((r = ioctl(kernel_fd, CRYPTO_DECRYPT, &decrypt)) < 0) {
243 		if (errno != EINTR)
244 			break;
245 	}
246 	if (r < 0) {
247 		rv = CKR_FUNCTION_FAILED;
248 	} else {
249 		rv = crypto2pkcs11_error_number(decrypt.cd_return_value);
250 	}
251 
252 	if (rv == CKR_OK || rv == CKR_BUFFER_TOO_SMALL)
253 		*pulDataLen = decrypt.cd_datalen;
254 
255 clean_exit:
256 
257 	if (ses_lock_held)
258 		(void) pthread_mutex_unlock(&session_p->session_mutex);
259 
260 	return (rv);
261 }
262 
263 CK_RV
264 C_Decrypt(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pEncryptedData,
265     CK_ULONG ulEncryptedData, CK_BYTE_PTR pData, CK_ULONG_PTR pulDataLen)
266 {
267 	CK_RV rv;
268 	kernel_session_t *session_p;
269 	boolean_t ses_lock_held = B_FALSE;
270 
271 	if (!kernel_initialized)
272 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
273 
274 	/* Obtain the session pointer. */
275 	rv = handle2session(hSession, &session_p);
276 	if (rv != CKR_OK)
277 		return (rv);
278 
279 	/*
280 	 * No need to check pData because application might
281 	 * just want to know the length of decrypted data.
282 	 */
283 	if (pulDataLen == NULL) {
284 		rv = CKR_ARGUMENTS_BAD;
285 		goto clean_exit;
286 	}
287 
288 	rv = kernel_decrypt(session_p, pEncryptedData, ulEncryptedData, pData,
289 	    pulDataLen);
290 
291 	if ((rv == CKR_BUFFER_TOO_SMALL) ||
292 	    (rv == CKR_OK && pData == NULL)) {
293 		/*
294 		 * We will not terminate the active decrypt operation flag,
295 		 * when the application-supplied buffer is too small, or
296 		 * the application asks for the length of buffer to hold
297 		 * the plaintext.
298 		 */
299 		REFRELE(session_p, ses_lock_held);
300 		return (rv);
301 	}
302 
303 clean_exit:
304 	/*
305 	 * Terminates the active decrypt operation.
306 	 * Application needs to call C_DecryptInit again for next
307 	 * decrypt operation.
308 	 */
309 	(void) pthread_mutex_lock(&session_p->session_mutex);
310 	session_p->decrypt.flags = 0;
311 	ses_lock_held = B_TRUE;
312 	REFRELE(session_p, ses_lock_held);
313 
314 	return (rv);
315 }
316 
317 
318 CK_RV
319 C_DecryptUpdate(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pEncryptedPart,
320     CK_ULONG ulEncryptedPartLen, CK_BYTE_PTR pPart,
321     CK_ULONG_PTR pulPartLen)
322 {
323 
324 	CK_RV rv;
325 	kernel_session_t *session_p;
326 	boolean_t ses_lock_held = B_FALSE;
327 	crypto_decrypt_update_t decrypt_update;
328 	int r;
329 
330 	if (!kernel_initialized)
331 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
332 
333 	/* Obtain the session pointer. */
334 	rv = handle2session(hSession, &session_p);
335 	if (rv != CKR_OK)
336 		return (rv);
337 
338 	if (pEncryptedPart == NULL) {
339 		rv = CKR_ARGUMENTS_BAD;
340 		goto clean_exit;
341 	}
342 
343 	/*
344 	 * Only check if pulPartLen is NULL.
345 	 * No need to check if pPart is NULL because application
346 	 * might just ask for the length of buffer to hold the
347 	 * recovered data.
348 	 */
349 	if (pulPartLen == NULL) {
350 		rv = CKR_ARGUMENTS_BAD;
351 		goto clean_exit;
352 	}
353 
354 	(void) pthread_mutex_lock(&session_p->session_mutex);
355 	ses_lock_held = B_TRUE;
356 
357 	/*
358 	 * Application must call C_DecryptInit before calling
359 	 * C_DecryptUpdate.
360 	 */
361 	if (!(session_p->decrypt.flags & CRYPTO_OPERATION_ACTIVE)) {
362 		REFRELE(session_p, ses_lock_held);
363 		return (CKR_OPERATION_NOT_INITIALIZED);
364 	}
365 
366 	session_p->decrypt.flags |= CRYPTO_OPERATION_UPDATE;
367 
368 	decrypt_update.du_session = session_p->k_session;
369 	(void) pthread_mutex_unlock(&session_p->session_mutex);
370 	ses_lock_held = B_FALSE;
371 
372 	decrypt_update.du_datalen = *pulPartLen;
373 	decrypt_update.du_databuf = (char *)pPart;
374 	decrypt_update.du_encrlen = ulEncryptedPartLen;
375 	decrypt_update.du_encrbuf = (char *)pEncryptedPart;
376 
377 	while ((r = ioctl(kernel_fd, CRYPTO_DECRYPT_UPDATE,
378 	    &decrypt_update)) < 0) {
379 		if (errno != EINTR)
380 			break;
381 	}
382 	if (r < 0) {
383 		rv = CKR_FUNCTION_FAILED;
384 	} else {
385 		rv = crypto2pkcs11_error_number(
386 		    decrypt_update.du_return_value);
387 	}
388 
389 	/*
390 	 * If CKR_OK or CKR_BUFFER_TOO_SMALL, set the output length.
391 	 * We don't terminate the current decryption operation.
392 	 */
393 	if (rv == CKR_OK || rv == CKR_BUFFER_TOO_SMALL) {
394 		*pulPartLen = decrypt_update.du_datalen;
395 		REFRELE(session_p, ses_lock_held);
396 		return (rv);
397 	}
398 
399 clean_exit:
400 	/*
401 	 * After an error occurred, terminate the current decrypt
402 	 * operation by resetting the active and update flags.
403 	 */
404 	(void) pthread_mutex_lock(&session_p->session_mutex);
405 	session_p->decrypt.flags = 0;
406 	ses_lock_held = B_TRUE;
407 	REFRELE(session_p, ses_lock_held);
408 
409 	return (rv);
410 }
411 
412 
413 CK_RV
414 C_DecryptFinal(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pLastPart,
415     CK_ULONG_PTR pulLastPartLen)
416 {
417 
418 	CK_RV rv;
419 	kernel_session_t *session_p;
420 	boolean_t ses_lock_held = B_FALSE;
421 	crypto_decrypt_final_t decrypt_final;
422 	int r;
423 
424 	if (!kernel_initialized)
425 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
426 
427 	/* Obtain the session pointer. */
428 	rv = handle2session(hSession, &session_p);
429 	if (rv != CKR_OK)
430 		return (rv);
431 
432 	if (pulLastPartLen == NULL) {
433 		rv = CKR_ARGUMENTS_BAD;
434 		goto clean_exit;
435 	}
436 
437 	(void) pthread_mutex_lock(&session_p->session_mutex);
438 	ses_lock_held = B_TRUE;
439 
440 	/*
441 	 * Application must call C_DecryptInit before calling
442 	 * C_DecryptFinal.
443 	 */
444 	if (!(session_p->decrypt.flags & CRYPTO_OPERATION_ACTIVE)) {
445 		REFRELE(session_p, ses_lock_held);
446 		return (CKR_OPERATION_NOT_INITIALIZED);
447 	}
448 
449 	decrypt_final.df_session = session_p->k_session;
450 	(void) pthread_mutex_unlock(&session_p->session_mutex);
451 	ses_lock_held = B_FALSE;
452 
453 	decrypt_final.df_datalen = *pulLastPartLen;
454 	decrypt_final.df_databuf = (char *)pLastPart;
455 
456 	while ((r = ioctl(kernel_fd, CRYPTO_DECRYPT_FINAL,
457 	    &decrypt_final)) < 0) {
458 		if (errno != EINTR)
459 			break;
460 	}
461 	if (r < 0) {
462 		rv = CKR_FUNCTION_FAILED;
463 	} else {
464 		rv = crypto2pkcs11_error_number(decrypt_final.df_return_value);
465 	}
466 
467 	if (rv == CKR_OK || rv == CKR_BUFFER_TOO_SMALL)
468 		*pulLastPartLen = decrypt_final.df_datalen;
469 
470 	if (rv == CKR_BUFFER_TOO_SMALL ||
471 	    (rv == CKR_OK && pLastPart == NULL)) {
472 		/*
473 		 * We will not terminate the active decrypt operation flag,
474 		 * when the application-supplied buffer is too small, or
475 		 * the application asks for the length of buffer to hold
476 		 * the plaintext.
477 		 */
478 		REFRELE(session_p, ses_lock_held);
479 		return (rv);
480 	}
481 
482 clean_exit:
483 	/* Terminates the active decrypt operation */
484 	(void) pthread_mutex_lock(&session_p->session_mutex);
485 	session_p->decrypt.flags = 0;
486 	ses_lock_held = B_TRUE;
487 	REFRELE(session_p, ses_lock_held);
488 
489 	return (rv);
490 }
491