xref: /titanic_50/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelEncrypt.c (revision c77a61a72b5ecdc507d6cf104142edd371a16c84)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 #include <pthread.h>
30 #include <stdlib.h>
31 #include <errno.h>
32 #include <sys/crypto/ioctl.h>
33 #include <security/cryptoki.h>
34 #include "kernelGlobal.h"
35 #include "kernelSession.h"
36 #include "kernelObject.h"
37 
38 
39 CK_RV
40 C_EncryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism,
41     CK_OBJECT_HANDLE hKey)
42 {
43 
44 	CK_RV rv;
45 	kernel_session_t *session_p;
46 	kernel_object_t	*key_p;
47 	boolean_t ses_lock_held = B_FALSE;
48 	crypto_encrypt_init_t encrypt_init;
49 	crypto_mech_type_t k_mech_type;
50 	int r;
51 
52 	if (!kernel_initialized)
53 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
54 
55 	if (pMechanism == NULL) {
56 		return (CKR_ARGUMENTS_BAD);
57 	}
58 
59 	/* Get the kernel's internal mechanism number. */
60 	rv = kernel_mech(pMechanism->mechanism, &k_mech_type);
61 	if (rv != CKR_OK)
62 		return (rv);
63 
64 	/* Obtain the session pointer. */
65 	rv = handle2session(hSession, &session_p);
66 	if (rv != CKR_OK)
67 		return (rv);
68 
69 	/* Obtain the object pointer. */
70 	HANDLE2OBJECT(hKey, key_p, rv);
71 	if (rv != CKR_OK) {
72 		REFRELE(session_p, ses_lock_held);
73 		return (rv);
74 	}
75 
76 	/* Check to see if key object allows for encryption. */
77 	if (key_p->is_lib_obj && !(key_p->bool_attr_mask & ENCRYPT_BOOL_ON)) {
78 		rv = CKR_KEY_TYPE_INCONSISTENT;
79 		goto clean_exit;
80 	}
81 
82 	(void) pthread_mutex_lock(&session_p->session_mutex);
83 	ses_lock_held = B_TRUE;
84 
85 	/*
86 	 * This active flag will remain ON until application calls either
87 	 * C_Encrypt or C_EncryptFinal to actually obtain the final piece
88 	 * of ciphertext.
89 	 */
90 	session_p->encrypt.flags = CRYPTO_OPERATION_ACTIVE;
91 
92 	/* set up key data */
93 	if (!key_p->is_lib_obj) {
94 		encrypt_init.ei_key.ck_format = CRYPTO_KEY_REFERENCE;
95 		encrypt_init.ei_key.ck_obj_id = key_p->k_handle;
96 	} else {
97 		if (key_p->class == CKO_SECRET_KEY) {
98 			encrypt_init.ei_key.ck_format = CRYPTO_KEY_RAW;
99 			encrypt_init.ei_key.ck_data =
100 			    get_symmetric_key_value(key_p);
101 			if (encrypt_init.ei_key.ck_data == NULL) {
102 				rv = CKR_HOST_MEMORY;
103 				goto clean_exit;
104 			}
105 			encrypt_init.ei_key.ck_length =
106 				OBJ_SEC(key_p)->sk_value_len << 3;
107 
108 		} else if (key_p->key_type == CKK_RSA) {
109 			if (get_rsa_public_key(key_p, &encrypt_init.ei_key) !=
110 			    CKR_OK) {
111 				rv = CKR_HOST_MEMORY;
112 				goto clean_exit;
113 			}
114 		} else {
115 			rv = CKR_KEY_TYPE_INCONSISTENT;
116 			goto clean_exit;
117 		}
118 	}
119 
120 	encrypt_init.ei_session = session_p->k_session;
121 	(void) pthread_mutex_unlock(&session_p->session_mutex);
122 	ses_lock_held = B_FALSE;
123 	encrypt_init.ei_mech.cm_type = k_mech_type;
124 	encrypt_init.ei_mech.cm_param = pMechanism->pParameter;
125 	encrypt_init.ei_mech.cm_param_len = pMechanism->ulParameterLen;
126 
127 	while ((r = ioctl(kernel_fd, CRYPTO_ENCRYPT_INIT, &encrypt_init)) < 0) {
128 		if (errno != EINTR)
129 			break;
130 	}
131 	if (r < 0) {
132 		rv = CKR_FUNCTION_FAILED;
133 	} else {
134 		if (encrypt_init.ei_return_value != CRYPTO_SUCCESS) {
135 			rv = crypto2pkcs11_error_number(
136 			    encrypt_init.ei_return_value);
137 		}
138 	}
139 
140 	/* Free memory allocated for decrypt_init.di_key */
141 	if (key_p->is_lib_obj) {
142 		if (key_p->class == CKO_SECRET_KEY) {
143 			free(encrypt_init.ei_key.ck_data);
144 		} else if (key_p->key_type == CKK_RSA) {
145 			free_key_attributes(&encrypt_init.ei_key);
146 		}
147 	}
148 
149 	if (rv != CKR_OK) {
150 		(void) pthread_mutex_lock(&session_p->session_mutex);
151 		session_p->encrypt.flags &= ~CRYPTO_OPERATION_ACTIVE;
152 		ses_lock_held = B_TRUE;
153 	}
154 
155 clean_exit:
156 	OBJ_REFRELE(key_p);
157 	REFRELE(session_p, ses_lock_held);
158 	return (rv);
159 }
160 
161 
162 CK_RV
163 C_Encrypt(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pData, CK_ULONG ulDataLen,
164     CK_BYTE_PTR pEncryptedData, CK_ULONG_PTR pulEncryptedDataLen)
165 {
166 
167 	CK_RV rv;
168 	kernel_session_t *session_p;
169 	boolean_t ses_lock_held = B_FALSE;
170 	crypto_encrypt_t encrypt;
171 	int r;
172 
173 	if (!kernel_initialized)
174 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
175 
176 	/* Obtain the session pointer. */
177 	rv = handle2session(hSession, &session_p);
178 	if (rv != CKR_OK)
179 		return (rv);
180 
181 	if (pData == NULL) {
182 		rv = CKR_ARGUMENTS_BAD;
183 		goto clean_exit;
184 	}
185 
186 	/*
187 	 * Only check if pulEncryptedDataLen is NULL.
188 	 * No need to check if pEncryptedData is NULL because
189 	 * application might just ask for the length of buffer to hold
190 	 * the ciphertext.
191 	 */
192 	if (pulEncryptedDataLen == NULL) {
193 		rv = CKR_ARGUMENTS_BAD;
194 		goto clean_exit;
195 	}
196 
197 	(void) pthread_mutex_lock(&session_p->session_mutex);
198 	ses_lock_held = B_TRUE;
199 
200 	/* Application must call C_EncryptInit before calling C_Encrypt. */
201 	if (!(session_p->encrypt.flags & CRYPTO_OPERATION_ACTIVE)) {
202 		REFRELE(session_p, ses_lock_held);
203 		return (CKR_OPERATION_NOT_INITIALIZED);
204 	}
205 
206 	/*
207 	 * C_Encrypt must be called without intervening C_EncryptUpdate
208 	 * calls.
209 	 */
210 	if (session_p->encrypt.flags & CRYPTO_OPERATION_UPDATE) {
211 		/*
212 		 * C_Encrypt can not be used to terminate a multi-part
213 		 * operation, so we'll leave the active encrypt operation
214 		 * flag on and let the application continue with the
215 		 * encrypt update operation.
216 		 */
217 		REFRELE(session_p, ses_lock_held);
218 		return (CKR_FUNCTION_FAILED);
219 	}
220 
221 	encrypt.ce_session = session_p->k_session;
222 	(void) pthread_mutex_unlock(&session_p->session_mutex);
223 	ses_lock_held = B_FALSE;
224 
225 	encrypt.ce_datalen = ulDataLen;
226 	encrypt.ce_databuf = (char *)pData;
227 	encrypt.ce_encrlen = *pulEncryptedDataLen;
228 	encrypt.ce_encrbuf = (char *)pEncryptedData;
229 
230 	while ((r = ioctl(kernel_fd, CRYPTO_ENCRYPT, &encrypt)) < 0) {
231 		if (errno != EINTR)
232 			break;
233 	}
234 	if (r < 0) {
235 		rv = CKR_FUNCTION_FAILED;
236 	} else {
237 		rv = crypto2pkcs11_error_number(encrypt.ce_return_value);
238 	}
239 
240 	if (rv == CKR_OK || rv == CKR_BUFFER_TOO_SMALL)
241 		*pulEncryptedDataLen = encrypt.ce_encrlen;
242 
243 	if ((rv == CKR_BUFFER_TOO_SMALL) ||
244 	    (rv == CKR_OK && pEncryptedData == NULL)) {
245 		/*
246 		 * We will not terminate the active encrypt operation flag,
247 		 * when the application-supplied buffer is too small, or
248 		 * the application asks for the length of buffer to hold
249 		 * the ciphertext.
250 		 */
251 		REFRELE(session_p, ses_lock_held);
252 		return (rv);
253 	}
254 
255 clean_exit:
256 	/*
257 	 * Terminates the active encrypt operation.
258 	 * Application needs to call C_EncryptInit again for next
259 	 * encrypt operation.
260 	 */
261 	(void) pthread_mutex_lock(&session_p->session_mutex);
262 	session_p->encrypt.flags = 0;
263 	ses_lock_held = B_TRUE;
264 	REFRELE(session_p, ses_lock_held);
265 
266 	return (rv);
267 }
268 
269 
270 CK_RV
271 C_EncryptUpdate(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pPart,
272     CK_ULONG ulPartLen, CK_BYTE_PTR pEncryptedPart,
273     CK_ULONG_PTR pulEncryptedPartLen)
274 {
275 
276 	CK_RV rv;
277 	kernel_session_t *session_p;
278 	boolean_t ses_lock_held = B_FALSE;
279 	crypto_encrypt_update_t encrypt_update;
280 	int r;
281 
282 	if (!kernel_initialized)
283 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
284 
285 	/* Obtain the session pointer. */
286 	rv = handle2session(hSession, &session_p);
287 	if (rv != CKR_OK)
288 		return (rv);
289 
290 	if (pPart == NULL) {
291 		rv = CKR_ARGUMENTS_BAD;
292 		goto clean_exit;
293 	}
294 
295 	/*
296 	 * Only check if pulEncryptedPartLen is NULL.
297 	 * No need to check if pEncryptedPart is NULL because
298 	 * application might just ask for the length of buffer to hold
299 	 * the ciphertext.
300 	 */
301 	if (pulEncryptedPartLen == NULL) {
302 		rv = CKR_ARGUMENTS_BAD;
303 		goto clean_exit;
304 	}
305 
306 	(void) pthread_mutex_lock(&session_p->session_mutex);
307 	ses_lock_held = B_TRUE;
308 
309 	/*
310 	 * Application must call C_EncryptInit before calling
311 	 * C_EncryptUpdate.
312 	 */
313 	if (!(session_p->encrypt.flags & CRYPTO_OPERATION_ACTIVE)) {
314 		REFRELE(session_p, ses_lock_held);
315 		return (CKR_OPERATION_NOT_INITIALIZED);
316 	}
317 
318 	session_p->encrypt.flags |= CRYPTO_OPERATION_UPDATE;
319 
320 	encrypt_update.eu_session = session_p->k_session;
321 	(void) pthread_mutex_unlock(&session_p->session_mutex);
322 	ses_lock_held = B_FALSE;
323 
324 	encrypt_update.eu_datalen = ulPartLen;
325 	encrypt_update.eu_databuf = (char *)pPart;
326 	encrypt_update.eu_encrlen = *pulEncryptedPartLen;
327 	encrypt_update.eu_encrbuf = (char *)pEncryptedPart;
328 
329 	while ((r = ioctl(kernel_fd, CRYPTO_ENCRYPT_UPDATE,
330 	    &encrypt_update)) < 0) {
331 		if (errno != EINTR)
332 			break;
333 	}
334 	if (r < 0) {
335 		rv = CKR_FUNCTION_FAILED;
336 	} else {
337 		rv = crypto2pkcs11_error_number(
338 		    encrypt_update.eu_return_value);
339 	}
340 
341 	/*
342 	 * If CKR_OK or CKR_BUFFER_TOO_SMALL, set the output length.
343 	 * We don't terminate the current encryption operation.
344 	 */
345 	if (rv == CKR_OK || rv == CKR_BUFFER_TOO_SMALL) {
346 		*pulEncryptedPartLen = encrypt_update.eu_encrlen;
347 		REFRELE(session_p, ses_lock_held);
348 		return (rv);
349 	}
350 
351 clean_exit:
352 	/*
353 	 * After an error occurred, terminate the current encrypt
354 	 * operation by resetting the active and update flags.
355 	 */
356 	(void) pthread_mutex_lock(&session_p->session_mutex);
357 	session_p->encrypt.flags = 0;
358 	ses_lock_held = B_TRUE;
359 	REFRELE(session_p, ses_lock_held);
360 
361 	return (rv);
362 }
363 
364 
365 CK_RV
366 C_EncryptFinal(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pLastEncryptedPart,
367     CK_ULONG_PTR pulLastEncryptedPartLen)
368 {
369 
370 	CK_RV rv;
371 	kernel_session_t *session_p;
372 	boolean_t ses_lock_held = B_FALSE;
373 	crypto_encrypt_final_t encrypt_final;
374 	int r;
375 
376 	if (!kernel_initialized)
377 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
378 
379 	/* Obtain the session pointer. */
380 	rv = handle2session(hSession, &session_p);
381 	if (rv != CKR_OK)
382 		return (rv);
383 
384 	if (pulLastEncryptedPartLen == NULL) {
385 		rv = CKR_ARGUMENTS_BAD;
386 		goto clean_exit;
387 	}
388 
389 	(void) pthread_mutex_lock(&session_p->session_mutex);
390 	ses_lock_held = B_TRUE;
391 
392 	/*
393 	 * Application must call C_EncryptInit before calling
394 	 * C_EncryptFinal.
395 	 */
396 	if (!(session_p->encrypt.flags & CRYPTO_OPERATION_ACTIVE)) {
397 		REFRELE(session_p, ses_lock_held);
398 		return (CKR_OPERATION_NOT_INITIALIZED);
399 	}
400 
401 	encrypt_final.ef_session = session_p->k_session;
402 	(void) pthread_mutex_unlock(&session_p->session_mutex);
403 	ses_lock_held = B_FALSE;
404 
405 	encrypt_final.ef_encrlen = *pulLastEncryptedPartLen;
406 	encrypt_final.ef_encrbuf = (char *)pLastEncryptedPart;
407 
408 	while ((r = ioctl(kernel_fd, CRYPTO_ENCRYPT_FINAL,
409 	    &encrypt_final)) < 0) {
410 		if (errno != EINTR)
411 			break;
412 	}
413 	if (r < 0) {
414 		rv = CKR_FUNCTION_FAILED;
415 	} else {
416 		rv = crypto2pkcs11_error_number(encrypt_final.ef_return_value);
417 	}
418 
419 	if (rv == CKR_OK || rv == CKR_BUFFER_TOO_SMALL)
420 		*pulLastEncryptedPartLen = encrypt_final.ef_encrlen;
421 
422 	if ((rv == CKR_BUFFER_TOO_SMALL) ||
423 	    (rv == CKR_OK && pLastEncryptedPart == NULL)) {
424 		/*
425 		 * We will not terminate the active encrypt operation flag,
426 		 * when the application-supplied buffer is too small, or
427 		 * the application asks for the length of buffer to hold
428 		 * the ciphertext.
429 		 */
430 		REFRELE(session_p, ses_lock_held);
431 		return (rv);
432 	}
433 
434 clean_exit:
435 	/* Terminates the active encrypt operation. */
436 	(void) pthread_mutex_lock(&session_p->session_mutex);
437 	session_p->encrypt.flags = 0;
438 	ses_lock_held = B_TRUE;
439 	REFRELE(session_p, ses_lock_held);
440 
441 	return (rv);
442 }
443