xref: /titanic_44/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelEncrypt.c (revision 09f67678c27dda8a89f87f1f408a87dd49ceb0e1)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 #include <pthread.h>
30 #include <stdlib.h>
31 #include <errno.h>
32 #include <sys/crypto/ioctl.h>
33 #include <security/cryptoki.h>
34 #include "kernelGlobal.h"
35 #include "kernelSession.h"
36 #include "kernelObject.h"
37 
38 
39 CK_RV
40 C_EncryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism,
41     CK_OBJECT_HANDLE hKey)
42 {
43 
44 	CK_RV rv;
45 	kernel_session_t *session_p;
46 	kernel_object_t	*key_p;
47 	boolean_t ses_lock_held = B_FALSE;
48 	crypto_encrypt_init_t encrypt_init;
49 	crypto_mech_type_t k_mech_type;
50 	int r;
51 
52 	if (!kernel_initialized)
53 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
54 
55 	if (pMechanism == NULL) {
56 		return (CKR_ARGUMENTS_BAD);
57 	}
58 
59 	/* Get the kernel's internal mechanism number. */
60 	rv = kernel_mech(pMechanism->mechanism, &k_mech_type);
61 	if (rv != CKR_OK)
62 		return (rv);
63 
64 	/* Obtain the session pointer. */
65 	rv = handle2session(hSession, &session_p);
66 	if (rv != CKR_OK)
67 		return (rv);
68 
69 	/* Obtain the object pointer. */
70 	HANDLE2OBJECT(hKey, key_p, rv);
71 	if (rv != CKR_OK)
72 		goto clean_exit;
73 
74 	/* Check to see if key object allows for encryption. */
75 	if (key_p->is_lib_obj && !(key_p->bool_attr_mask & ENCRYPT_BOOL_ON)) {
76 		rv = CKR_KEY_TYPE_INCONSISTENT;
77 		goto clean_exit;
78 	}
79 
80 	(void) pthread_mutex_lock(&session_p->session_mutex);
81 	ses_lock_held = B_TRUE;
82 
83 	/*
84 	 * This active flag will remain ON until application calls either
85 	 * C_Encrypt or C_EncryptFinal to actually obtain the final piece
86 	 * of ciphertext.
87 	 */
88 	session_p->encrypt.flags = CRYPTO_OPERATION_ACTIVE;
89 
90 	/* set up key data */
91 	if (!key_p->is_lib_obj) {
92 		encrypt_init.ei_key.ck_format = CRYPTO_KEY_REFERENCE;
93 		encrypt_init.ei_key.ck_obj_id = key_p->k_handle;
94 	} else {
95 		if (key_p->class == CKO_SECRET_KEY) {
96 			encrypt_init.ei_key.ck_format = CRYPTO_KEY_RAW;
97 			encrypt_init.ei_key.ck_data =
98 			    get_symmetric_key_value(key_p);
99 			if (encrypt_init.ei_key.ck_data == NULL) {
100 				rv = CKR_HOST_MEMORY;
101 				goto clean_exit;
102 			}
103 			encrypt_init.ei_key.ck_length =
104 				OBJ_SEC(key_p)->sk_value_len << 3;
105 
106 		} else if (key_p->key_type == CKK_RSA) {
107 			if (get_rsa_public_key(key_p, &encrypt_init.ei_key) !=
108 			    CKR_OK) {
109 				rv = CKR_HOST_MEMORY;
110 				goto clean_exit;
111 			}
112 		} else {
113 			rv = CKR_KEY_TYPE_INCONSISTENT;
114 			goto clean_exit;
115 		}
116 	}
117 
118 	encrypt_init.ei_session = session_p->k_session;
119 	(void) pthread_mutex_unlock(&session_p->session_mutex);
120 	ses_lock_held = B_FALSE;
121 	encrypt_init.ei_mech.cm_type = k_mech_type;
122 	encrypt_init.ei_mech.cm_param = pMechanism->pParameter;
123 	encrypt_init.ei_mech.cm_param_len = pMechanism->ulParameterLen;
124 
125 	while ((r = ioctl(kernel_fd, CRYPTO_ENCRYPT_INIT, &encrypt_init)) < 0) {
126 		if (errno != EINTR)
127 			break;
128 	}
129 	if (r < 0) {
130 		rv = CKR_FUNCTION_FAILED;
131 	} else {
132 		if (encrypt_init.ei_return_value != CRYPTO_SUCCESS) {
133 			rv = crypto2pkcs11_error_number(
134 			    encrypt_init.ei_return_value);
135 		}
136 	}
137 
138 	/* Free memory allocated for decrypt_init.di_key */
139 	if (key_p->is_lib_obj) {
140 		if (key_p->class == CKO_SECRET_KEY) {
141 			free(encrypt_init.ei_key.ck_data);
142 		} else if (key_p->key_type == CKK_RSA) {
143 			free_key_attributes(&encrypt_init.ei_key);
144 		}
145 	}
146 
147 	if (rv != CKR_OK) {
148 		(void) pthread_mutex_lock(&session_p->session_mutex);
149 		session_p->encrypt.flags &= ~CRYPTO_OPERATION_ACTIVE;
150 		ses_lock_held = B_TRUE;
151 	}
152 
153 clean_exit:
154 	REFRELE(session_p, ses_lock_held);
155 	return (rv);
156 }
157 
158 
159 CK_RV
160 C_Encrypt(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pData, CK_ULONG ulDataLen,
161     CK_BYTE_PTR pEncryptedData, CK_ULONG_PTR pulEncryptedDataLen)
162 {
163 
164 	CK_RV rv;
165 	kernel_session_t *session_p;
166 	boolean_t ses_lock_held = B_FALSE;
167 	crypto_encrypt_t encrypt;
168 	int r;
169 
170 	if (!kernel_initialized)
171 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
172 
173 	/* Obtain the session pointer. */
174 	rv = handle2session(hSession, &session_p);
175 	if (rv != CKR_OK)
176 		return (rv);
177 
178 	if (pData == NULL) {
179 		rv = CKR_ARGUMENTS_BAD;
180 		goto clean_exit;
181 	}
182 
183 	/*
184 	 * Only check if pulEncryptedDataLen is NULL.
185 	 * No need to check if pEncryptedData is NULL because
186 	 * application might just ask for the length of buffer to hold
187 	 * the ciphertext.
188 	 */
189 	if (pulEncryptedDataLen == NULL) {
190 		rv = CKR_ARGUMENTS_BAD;
191 		goto clean_exit;
192 	}
193 
194 	(void) pthread_mutex_lock(&session_p->session_mutex);
195 	ses_lock_held = B_TRUE;
196 
197 	/* Application must call C_EncryptInit before calling C_Encrypt. */
198 	if (!(session_p->encrypt.flags & CRYPTO_OPERATION_ACTIVE)) {
199 		REFRELE(session_p, ses_lock_held);
200 		return (CKR_OPERATION_NOT_INITIALIZED);
201 	}
202 
203 	/*
204 	 * C_Encrypt must be called without intervening C_EncryptUpdate
205 	 * calls.
206 	 */
207 	if (session_p->encrypt.flags & CRYPTO_OPERATION_UPDATE) {
208 		/*
209 		 * C_Encrypt can not be used to terminate a multi-part
210 		 * operation, so we'll leave the active encrypt operation
211 		 * flag on and let the application continue with the
212 		 * encrypt update operation.
213 		 */
214 		REFRELE(session_p, ses_lock_held);
215 		return (CKR_FUNCTION_FAILED);
216 	}
217 
218 	encrypt.ce_session = session_p->k_session;
219 	(void) pthread_mutex_unlock(&session_p->session_mutex);
220 	ses_lock_held = B_FALSE;
221 
222 	encrypt.ce_datalen = ulDataLen;
223 	encrypt.ce_databuf = (char *)pData;
224 	encrypt.ce_encrlen = *pulEncryptedDataLen;
225 	encrypt.ce_encrbuf = (char *)pEncryptedData;
226 
227 	while ((r = ioctl(kernel_fd, CRYPTO_ENCRYPT, &encrypt)) < 0) {
228 		if (errno != EINTR)
229 			break;
230 	}
231 	if (r < 0) {
232 		rv = CKR_FUNCTION_FAILED;
233 	} else {
234 		rv = crypto2pkcs11_error_number(encrypt.ce_return_value);
235 	}
236 
237 	if (rv == CKR_OK || rv == CKR_BUFFER_TOO_SMALL)
238 		*pulEncryptedDataLen = encrypt.ce_encrlen;
239 
240 	if ((rv == CKR_BUFFER_TOO_SMALL) ||
241 	    (rv == CKR_OK && pEncryptedData == NULL)) {
242 		/*
243 		 * We will not terminate the active encrypt operation flag,
244 		 * when the application-supplied buffer is too small, or
245 		 * the application asks for the length of buffer to hold
246 		 * the ciphertext.
247 		 */
248 		REFRELE(session_p, ses_lock_held);
249 		return (rv);
250 	}
251 
252 clean_exit:
253 	/*
254 	 * Terminates the active encrypt operation.
255 	 * Application needs to call C_EncryptInit again for next
256 	 * encrypt operation.
257 	 */
258 	(void) pthread_mutex_lock(&session_p->session_mutex);
259 	session_p->encrypt.flags = 0;
260 	ses_lock_held = B_TRUE;
261 	REFRELE(session_p, ses_lock_held);
262 
263 	return (rv);
264 }
265 
266 
267 CK_RV
268 C_EncryptUpdate(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pPart,
269     CK_ULONG ulPartLen, CK_BYTE_PTR pEncryptedPart,
270     CK_ULONG_PTR pulEncryptedPartLen)
271 {
272 
273 	CK_RV rv;
274 	kernel_session_t *session_p;
275 	boolean_t ses_lock_held = B_FALSE;
276 	crypto_encrypt_update_t encrypt_update;
277 	int r;
278 
279 	if (!kernel_initialized)
280 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
281 
282 	/* Obtain the session pointer. */
283 	rv = handle2session(hSession, &session_p);
284 	if (rv != CKR_OK)
285 		return (rv);
286 
287 	if (pPart == NULL) {
288 		rv = CKR_ARGUMENTS_BAD;
289 		goto clean_exit;
290 	}
291 
292 	/*
293 	 * Only check if pulEncryptedPartLen is NULL.
294 	 * No need to check if pEncryptedPart is NULL because
295 	 * application might just ask for the length of buffer to hold
296 	 * the ciphertext.
297 	 */
298 	if (pulEncryptedPartLen == NULL) {
299 		rv = CKR_ARGUMENTS_BAD;
300 		goto clean_exit;
301 	}
302 
303 	(void) pthread_mutex_lock(&session_p->session_mutex);
304 	ses_lock_held = B_TRUE;
305 
306 	/*
307 	 * Application must call C_EncryptInit before calling
308 	 * C_EncryptUpdate.
309 	 */
310 	if (!(session_p->encrypt.flags & CRYPTO_OPERATION_ACTIVE)) {
311 		REFRELE(session_p, ses_lock_held);
312 		return (CKR_OPERATION_NOT_INITIALIZED);
313 	}
314 
315 	session_p->encrypt.flags |= CRYPTO_OPERATION_UPDATE;
316 
317 	encrypt_update.eu_session = session_p->k_session;
318 	(void) pthread_mutex_unlock(&session_p->session_mutex);
319 	ses_lock_held = B_FALSE;
320 
321 	encrypt_update.eu_datalen = ulPartLen;
322 	encrypt_update.eu_databuf = (char *)pPart;
323 	encrypt_update.eu_encrlen = *pulEncryptedPartLen;
324 	encrypt_update.eu_encrbuf = (char *)pEncryptedPart;
325 
326 	while ((r = ioctl(kernel_fd, CRYPTO_ENCRYPT_UPDATE,
327 	    &encrypt_update)) < 0) {
328 		if (errno != EINTR)
329 			break;
330 	}
331 	if (r < 0) {
332 		rv = CKR_FUNCTION_FAILED;
333 	} else {
334 		rv = crypto2pkcs11_error_number(
335 		    encrypt_update.eu_return_value);
336 	}
337 
338 	/*
339 	 * If CKR_OK or CKR_BUFFER_TOO_SMALL, set the output length.
340 	 * We don't terminate the current encryption operation.
341 	 */
342 	if (rv == CKR_OK || rv == CKR_BUFFER_TOO_SMALL) {
343 		*pulEncryptedPartLen = encrypt_update.eu_encrlen;
344 		REFRELE(session_p, ses_lock_held);
345 		return (rv);
346 	}
347 
348 clean_exit:
349 	/*
350 	 * After an error occurred, terminate the current encrypt
351 	 * operation by resetting the active and update flags.
352 	 */
353 	(void) pthread_mutex_lock(&session_p->session_mutex);
354 	session_p->encrypt.flags = 0;
355 	ses_lock_held = B_TRUE;
356 	REFRELE(session_p, ses_lock_held);
357 
358 	return (rv);
359 }
360 
361 
362 CK_RV
363 C_EncryptFinal(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pLastEncryptedPart,
364     CK_ULONG_PTR pulLastEncryptedPartLen)
365 {
366 
367 	CK_RV rv;
368 	kernel_session_t *session_p;
369 	boolean_t ses_lock_held = B_FALSE;
370 	crypto_encrypt_final_t encrypt_final;
371 	int r;
372 
373 	if (!kernel_initialized)
374 		return (CKR_CRYPTOKI_NOT_INITIALIZED);
375 
376 	/* Obtain the session pointer. */
377 	rv = handle2session(hSession, &session_p);
378 	if (rv != CKR_OK)
379 		return (rv);
380 
381 	if (pulLastEncryptedPartLen == NULL) {
382 		rv = CKR_ARGUMENTS_BAD;
383 		goto clean_exit;
384 	}
385 
386 	(void) pthread_mutex_lock(&session_p->session_mutex);
387 	ses_lock_held = B_TRUE;
388 
389 	/*
390 	 * Application must call C_EncryptInit before calling
391 	 * C_EncryptFinal.
392 	 */
393 	if (!(session_p->encrypt.flags & CRYPTO_OPERATION_ACTIVE)) {
394 		REFRELE(session_p, ses_lock_held);
395 		return (CKR_OPERATION_NOT_INITIALIZED);
396 	}
397 
398 	encrypt_final.ef_session = session_p->k_session;
399 	(void) pthread_mutex_unlock(&session_p->session_mutex);
400 	ses_lock_held = B_FALSE;
401 
402 	encrypt_final.ef_encrlen = *pulLastEncryptedPartLen;
403 	encrypt_final.ef_encrbuf = (char *)pLastEncryptedPart;
404 
405 	while ((r = ioctl(kernel_fd, CRYPTO_ENCRYPT_FINAL,
406 	    &encrypt_final)) < 0) {
407 		if (errno != EINTR)
408 			break;
409 	}
410 	if (r < 0) {
411 		rv = CKR_FUNCTION_FAILED;
412 	} else {
413 		rv = crypto2pkcs11_error_number(encrypt_final.ef_return_value);
414 	}
415 
416 	if (rv == CKR_OK || rv == CKR_BUFFER_TOO_SMALL)
417 		*pulLastEncryptedPartLen = encrypt_final.ef_encrlen;
418 
419 	if ((rv == CKR_BUFFER_TOO_SMALL) ||
420 	    (rv == CKR_OK && pLastEncryptedPart == NULL)) {
421 		/*
422 		 * We will not terminate the active encrypt operation flag,
423 		 * when the application-supplied buffer is too small, or
424 		 * the application asks for the length of buffer to hold
425 		 * the ciphertext.
426 		 */
427 		REFRELE(session_p, ses_lock_held);
428 		return (rv);
429 	}
430 
431 clean_exit:
432 	/* Terminates the active encrypt operation. */
433 	(void) pthread_mutex_lock(&session_p->session_mutex);
434 	session_p->encrypt.flags = 0;
435 	ses_lock_held = B_TRUE;
436 	REFRELE(session_p, ses_lock_held);
437 
438 	return (rv);
439 }
440