xref: /titanic_50/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelEncrypt.c (revision 6d2259e1baf8d4ac11c96570f45ecdcd9771a68d)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <pthread.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/crypto/ioctl.h>
#include <security/cryptoki.h>
#include "kernelGlobal.h"
#include "kernelSession.h"
#include "kernelObject.h"


CK_RV
C_EncryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism,
    CK_OBJECT_HANDLE hKey)
{

	CK_RV rv;
	kernel_session_t *session_p;
	kernel_object_t	*key_p;
	boolean_t ses_lock_held = B_FALSE;
	crypto_encrypt_init_t encrypt_init;
	crypto_mech_type_t k_mech_type;
	int r;

	if (!kernel_initialized)
		return (CKR_CRYPTOKI_NOT_INITIALIZED);

	if (pMechanism == NULL) {
		return (CKR_ARGUMENTS_BAD);
	}

	/* Get the kernel's internal mechanism number. */
	rv = kernel_mech(pMechanism->mechanism, &k_mech_type);
	if (rv != CKR_OK)
		return (rv);

	/* Obtain the session pointer. */
	rv = handle2session(hSession, &session_p);
	if (rv != CKR_OK)
		return (rv);

	/* Obtain the object pointer. */
	HANDLE2OBJECT(hKey, key_p, rv);
	if (rv != CKR_OK) {
		REFRELE(session_p, ses_lock_held);
		return (rv);
	}

	/* Check to see if key object allows for encryption. */
	if (key_p->is_lib_obj && !(key_p->bool_attr_mask & ENCRYPT_BOOL_ON)) {
		rv = CKR_KEY_TYPE_INCONSISTENT;
		goto clean_exit;
	}

	(void) pthread_mutex_lock(&session_p->session_mutex);
	ses_lock_held = B_TRUE;

	/*
	 * This active flag will remain ON until application calls either
	 * C_Encrypt or C_EncryptFinal to actually obtain the final piece
	 * of ciphertext.
	 */
	session_p->encrypt.flags = CRYPTO_OPERATION_ACTIVE;

	/* set up key data */
	if (!key_p->is_lib_obj) {
		encrypt_init.ei_key.ck_format = CRYPTO_KEY_REFERENCE;
		encrypt_init.ei_key.ck_obj_id = key_p->k_handle;
	} else {
		if (key_p->class == CKO_SECRET_KEY) {
			encrypt_init.ei_key.ck_format = CRYPTO_KEY_RAW;
			encrypt_init.ei_key.ck_data =
			    get_symmetric_key_value(key_p);
			if (encrypt_init.ei_key.ck_data == NULL) {
				rv = CKR_HOST_MEMORY;
				goto clean_exit;
			}
			encrypt_init.ei_key.ck_length =
			    OBJ_SEC(key_p)->sk_value_len << 3;

		} else if (key_p->key_type == CKK_RSA) {
			if (get_rsa_public_key(key_p, &encrypt_init.ei_key) !=
			    CKR_OK) {
				rv = CKR_HOST_MEMORY;
				goto clean_exit;
			}
		} else {
			rv = CKR_KEY_TYPE_INCONSISTENT;
			goto clean_exit;
		}
	}

	encrypt_init.ei_session = session_p->k_session;
	session_p->encrypt.mech = *pMechanism;

	/* Cache this capability value for efficiency */
	if (INPLACE_MECHANISM(session_p->encrypt.mech.mechanism)) {
		session_p->encrypt.flags |= CRYPTO_OPERATION_INPLACE_OK;
	}
	(void) pthread_mutex_unlock(&session_p->session_mutex);

	ses_lock_held = B_FALSE;
	encrypt_init.ei_mech.cm_type = k_mech_type;
	encrypt_init.ei_mech.cm_param = pMechanism->pParameter;
	encrypt_init.ei_mech.cm_param_len = pMechanism->ulParameterLen;

	while ((r = ioctl(kernel_fd, CRYPTO_ENCRYPT_INIT, &encrypt_init)) < 0) {
		if (errno != EINTR)
			break;
	}
	if (r < 0) {
		rv = CKR_FUNCTION_FAILED;
	} else {
		if (encrypt_init.ei_return_value != CRYPTO_SUCCESS) {
			rv = crypto2pkcs11_error_number(
			    encrypt_init.ei_return_value);
		}
	}

	/* Free memory allocated for decrypt_init.di_key */
	if (key_p->is_lib_obj) {
		if (key_p->class == CKO_SECRET_KEY) {
			free(encrypt_init.ei_key.ck_data);
		} else if (key_p->key_type == CKK_RSA) {
			free_key_attributes(&encrypt_init.ei_key);
		}
	}

	if (rv != CKR_OK) {
		(void) pthread_mutex_lock(&session_p->session_mutex);
		session_p->encrypt.flags &= ~CRYPTO_OPERATION_ACTIVE;
		ses_lock_held = B_TRUE;
	}

clean_exit:
	OBJ_REFRELE(key_p);
	REFRELE(session_p, ses_lock_held);
	return (rv);
}


CK_RV
C_Encrypt(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pData, CK_ULONG ulDataLen,
    CK_BYTE_PTR pEncryptedData, CK_ULONG_PTR pulEncryptedDataLen)
{

	CK_RV rv;
	kernel_session_t *session_p;
	boolean_t ses_lock_held = B_FALSE;
	boolean_t inplace;
	crypto_encrypt_t encrypt;
	int r;

	if (!kernel_initialized)
		return (CKR_CRYPTOKI_NOT_INITIALIZED);

	/* Obtain the session pointer. */
	rv = handle2session(hSession, &session_p);
	if (rv != CKR_OK)
		return (rv);

	if (pData == NULL) {
		rv = CKR_ARGUMENTS_BAD;
		goto clean_exit;
	}

	/*
	 * Only check if pulEncryptedDataLen is NULL.
	 * No need to check if pEncryptedData is NULL because
	 * application might just ask for the length of buffer to hold
	 * the ciphertext.
	 */
	if (pulEncryptedDataLen == NULL) {
		rv = CKR_ARGUMENTS_BAD;
		goto clean_exit;
	}

	(void) pthread_mutex_lock(&session_p->session_mutex);
	ses_lock_held = B_TRUE;

	/* Application must call C_EncryptInit before calling C_Encrypt. */
	if (!(session_p->encrypt.flags & CRYPTO_OPERATION_ACTIVE)) {
		REFRELE(session_p, ses_lock_held);
		return (CKR_OPERATION_NOT_INITIALIZED);
	}

	/*
	 * C_Encrypt must be called without intervening C_EncryptUpdate
	 * calls.
	 */
	if (session_p->encrypt.flags & CRYPTO_OPERATION_UPDATE) {
		/*
		 * C_Encrypt can not be used to terminate a multi-part
		 * operation, so we'll leave the active encrypt operation
		 * flag on and let the application continue with the
		 * encrypt update operation.
		 */
		REFRELE(session_p, ses_lock_held);
		return (CKR_FUNCTION_FAILED);
	}

	encrypt.ce_session = session_p->k_session;

	/*
	 * Certain mechanisms, where the length of the ciphertext is
	 * same as the transformed plaintext, can be optimized
	 * by the kernel into an in-place operation. Unfortunately,
	 * some applications use a ciphertext buffer that is larger
	 * than it needs to be. We fix that here.
	 */
	inplace = (session_p->encrypt.flags & CRYPTO_OPERATION_INPLACE_OK) != 0;
	if (ulDataLen < *pulEncryptedDataLen && inplace) {
		encrypt.ce_encrlen = ulDataLen;
	} else {
		encrypt.ce_encrlen = *pulEncryptedDataLen;
	}
	(void) pthread_mutex_unlock(&session_p->session_mutex);
	ses_lock_held = B_FALSE;

	encrypt.ce_datalen = ulDataLen;
	encrypt.ce_databuf = (char *)pData;
	encrypt.ce_encrbuf = (char *)pEncryptedData;
	encrypt.ce_flags =
	    ((inplace && (pEncryptedData != NULL)) ||
	    (pData == pEncryptedData)) &&
	    (encrypt.ce_encrlen == encrypt.ce_datalen) ?
	    CRYPTO_INPLACE_OPERATION : 0;

	while ((r = ioctl(kernel_fd, CRYPTO_ENCRYPT, &encrypt)) < 0) {
		if (errno != EINTR)
			break;
	}
	if (r < 0) {
		rv = CKR_FUNCTION_FAILED;
	} else {
		rv = crypto2pkcs11_error_number(encrypt.ce_return_value);
	}

	if (rv == CKR_OK || rv == CKR_BUFFER_TOO_SMALL)
		*pulEncryptedDataLen = encrypt.ce_encrlen;

	if ((rv == CKR_BUFFER_TOO_SMALL) ||
	    (rv == CKR_OK && pEncryptedData == NULL)) {
		/*
		 * We will not terminate the active encrypt operation flag,
		 * when the application-supplied buffer is too small, or
		 * the application asks for the length of buffer to hold
		 * the ciphertext.
		 */
		REFRELE(session_p, ses_lock_held);
		return (rv);
	}

clean_exit:
	/*
	 * Terminates the active encrypt operation.
	 * Application needs to call C_EncryptInit again for next
	 * encrypt operation.
	 */
	(void) pthread_mutex_lock(&session_p->session_mutex);
	session_p->encrypt.flags = 0;
	ses_lock_held = B_TRUE;
	REFRELE(session_p, ses_lock_held);

	return (rv);
}


CK_RV
C_EncryptUpdate(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pPart,
    CK_ULONG ulPartLen, CK_BYTE_PTR pEncryptedPart,
    CK_ULONG_PTR pulEncryptedPartLen)
{

	CK_RV rv;
	kernel_session_t *session_p;
	boolean_t ses_lock_held = B_FALSE;
	boolean_t inplace;
	crypto_encrypt_update_t encrypt_update;
	int r;

	if (!kernel_initialized)
		return (CKR_CRYPTOKI_NOT_INITIALIZED);

	/* Obtain the session pointer. */
	rv = handle2session(hSession, &session_p);
	if (rv != CKR_OK)
		return (rv);

	if (pPart == NULL) {
		rv = CKR_ARGUMENTS_BAD;
		goto clean_exit;
	}

	/*
	 * Only check if pulEncryptedPartLen is NULL.
	 * No need to check if pEncryptedPart is NULL because
	 * application might just ask for the length of buffer to hold
	 * the ciphertext.
	 */
	if (pulEncryptedPartLen == NULL) {
		rv = CKR_ARGUMENTS_BAD;
		goto clean_exit;
	}

	(void) pthread_mutex_lock(&session_p->session_mutex);
	ses_lock_held = B_TRUE;

	/*
	 * Application must call C_EncryptInit before calling
	 * C_EncryptUpdate.
	 */
	if (!(session_p->encrypt.flags & CRYPTO_OPERATION_ACTIVE)) {
		REFRELE(session_p, ses_lock_held);
		return (CKR_OPERATION_NOT_INITIALIZED);
	}

	session_p->encrypt.flags |= CRYPTO_OPERATION_UPDATE;

	encrypt_update.eu_session = session_p->k_session;
	(void) pthread_mutex_unlock(&session_p->session_mutex);
	ses_lock_held = B_FALSE;

	encrypt_update.eu_datalen = ulPartLen;
	encrypt_update.eu_databuf = (char *)pPart;
	encrypt_update.eu_encrlen = *pulEncryptedPartLen;
	encrypt_update.eu_encrbuf = (char *)pEncryptedPart;

	inplace = (session_p->encrypt.flags & CRYPTO_OPERATION_INPLACE_OK) != 0;
	encrypt_update.eu_flags =
	    ((inplace && (pEncryptedPart != NULL)) ||
	    (pPart == pEncryptedPart)) &&
	    (encrypt_update.eu_encrlen == encrypt_update.eu_datalen) ?
	    CRYPTO_INPLACE_OPERATION : 0;

	while ((r = ioctl(kernel_fd, CRYPTO_ENCRYPT_UPDATE,
	    &encrypt_update)) < 0) {
		if (errno != EINTR)
			break;
	}
	if (r < 0) {
		rv = CKR_FUNCTION_FAILED;
	} else {
		rv = crypto2pkcs11_error_number(
		    encrypt_update.eu_return_value);
	}

	/*
	 * If CKR_OK or CKR_BUFFER_TOO_SMALL, set the output length.
	 * We don't terminate the current encryption operation.
	 */
	if (rv == CKR_OK || rv == CKR_BUFFER_TOO_SMALL) {
		*pulEncryptedPartLen = encrypt_update.eu_encrlen;
		REFRELE(session_p, ses_lock_held);
		return (rv);
	}

clean_exit:
	/*
	 * After an error occurred, terminate the current encrypt
	 * operation by resetting the active and update flags.
	 */
	(void) pthread_mutex_lock(&session_p->session_mutex);
	session_p->encrypt.flags = 0;
	ses_lock_held = B_TRUE;
	REFRELE(session_p, ses_lock_held);

	return (rv);
}


CK_RV
C_EncryptFinal(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pLastEncryptedPart,
    CK_ULONG_PTR pulLastEncryptedPartLen)
{

	CK_RV rv;
	kernel_session_t *session_p;
	boolean_t ses_lock_held = B_FALSE;
	crypto_encrypt_final_t encrypt_final;
	int r;

	if (!kernel_initialized)
		return (CKR_CRYPTOKI_NOT_INITIALIZED);

	/* Obtain the session pointer. */
	rv = handle2session(hSession, &session_p);
	if (rv != CKR_OK)
		return (rv);

	if (pulLastEncryptedPartLen == NULL) {
		rv = CKR_ARGUMENTS_BAD;
		goto clean_exit;
	}

	(void) pthread_mutex_lock(&session_p->session_mutex);
	ses_lock_held = B_TRUE;

	/*
	 * Application must call C_EncryptInit before calling
	 * C_EncryptFinal.
	 */
	if (!(session_p->encrypt.flags & CRYPTO_OPERATION_ACTIVE)) {
		REFRELE(session_p, ses_lock_held);
		return (CKR_OPERATION_NOT_INITIALIZED);
	}

	encrypt_final.ef_session = session_p->k_session;
	(void) pthread_mutex_unlock(&session_p->session_mutex);
	ses_lock_held = B_FALSE;

	encrypt_final.ef_encrlen = *pulLastEncryptedPartLen;
	encrypt_final.ef_encrbuf = (char *)pLastEncryptedPart;

	while ((r = ioctl(kernel_fd, CRYPTO_ENCRYPT_FINAL,
	    &encrypt_final)) < 0) {
		if (errno != EINTR)
			break;
	}
	if (r < 0) {
		rv = CKR_FUNCTION_FAILED;
	} else {
		rv = crypto2pkcs11_error_number(encrypt_final.ef_return_value);
	}

	if (rv == CKR_OK || rv == CKR_BUFFER_TOO_SMALL)
		*pulLastEncryptedPartLen = encrypt_final.ef_encrlen;

	if ((rv == CKR_BUFFER_TOO_SMALL) ||
	    (rv == CKR_OK && pLastEncryptedPart == NULL)) {
		/*
		 * We will not terminate the active encrypt operation flag,
		 * when the application-supplied buffer is too small, or
		 * the application asks for the length of buffer to hold
		 * the ciphertext.
		 */
		REFRELE(session_p, ses_lock_held);
		return (rv);
	}

clean_exit:
	/* Terminates the active encrypt operation. */
	(void) pthread_mutex_lock(&session_p->session_mutex);
	session_p->encrypt.flags = 0;
	ses_lock_held = B_TRUE;
	REFRELE(session_p, ses_lock_held);

	return (rv);
}