xref: /titanic_41/usr/src/lib/pkcs11/pkcs11_softtoken/common/softRSA.c (revision 0b42f15ac52b077791d4ba079e8c163c592c3fda)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
 */

#include <pthread.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <sys/types.h>
#include <security/cryptoki.h>
#include <cryptoutil.h>
#include "softGlobal.h"
#include "softSession.h"
#include "softObject.h"
#include "softOps.h"
#include "softRSA.h"
#include "softMAC.h"
#include "softCrypt.h"

CK_RV
soft_rsa_encrypt(soft_object_t *key, CK_BYTE_PTR in, uint32_t in_len,
    CK_BYTE_PTR out, int realpublic)
{

	CK_RV rv = CKR_OK;

	uchar_t expo[MAX_KEY_ATTR_BUFLEN];
	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
	uint32_t expo_len = sizeof (expo);
	uint32_t modulus_len = sizeof (modulus);
	RSAbytekey k;

	if (realpublic) {
		rv = soft_get_public_value(key, CKA_PUBLIC_EXPONENT, expo,
		    &expo_len);
		if (rv != CKR_OK) {
			goto clean1;
		}
	} else {
		rv = soft_get_private_value(key, CKA_PRIVATE_EXPONENT, expo,
		    &expo_len);
		if (rv != CKR_OK) {
			goto clean1;
		}
	}

	rv = soft_get_public_value(key, CKA_MODULUS, modulus, &modulus_len);
	if (rv != CKR_OK) {
		goto clean1;
	}

	k.modulus = modulus;
	k.modulus_bits = CRYPTO_BYTES2BITS(modulus_len);
	k.pubexpo = expo;
	k.pubexpo_bytes = expo_len;
	k.rfunc = NULL;

	rv = rsa_encrypt(&k, in, in_len, out);

clean1:

	return (rv);
}


CK_RV
soft_rsa_decrypt(soft_object_t *key, CK_BYTE_PTR in, uint32_t in_len,
    CK_BYTE_PTR out)
{

	CK_RV rv = CKR_OK;

	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
	uchar_t prime1[MAX_KEY_ATTR_BUFLEN];
	uchar_t prime2[MAX_KEY_ATTR_BUFLEN];
	uchar_t expo1[MAX_KEY_ATTR_BUFLEN];
	uchar_t expo2[MAX_KEY_ATTR_BUFLEN];
	uchar_t coef[MAX_KEY_ATTR_BUFLEN];
	uint32_t modulus_len = sizeof (modulus);
	uint32_t prime1_len = sizeof (prime1);
	uint32_t prime2_len = sizeof (prime2);
	uint32_t expo1_len = sizeof (expo1);
	uint32_t expo2_len = sizeof (expo2);
	uint32_t coef_len = sizeof (coef);
	RSAbytekey k;

	rv = soft_get_private_value(key, CKA_MODULUS, modulus, &modulus_len);
	if (rv != CKR_OK) {
		goto clean1;
	}

	rv = soft_get_private_value(key, CKA_PRIME_1, prime1, &prime1_len);

	if ((prime1_len == 0) && (rv == CKR_OK)) {
		rv = soft_rsa_encrypt(key, in, in_len, out, 0);
		goto clean1;
	} else {
		if (rv != CKR_OK)
			goto clean1;
	}

	rv = soft_get_private_value(key, CKA_PRIME_2, prime2, &prime2_len);

	if ((prime2_len == 0) && (rv == CKR_OK)) {
		rv = soft_rsa_encrypt(key, in, in_len, out, 0);
		goto clean1;
	} else {
		if (rv != CKR_OK)
			goto clean1;
	}

	rv = soft_get_private_value(key, CKA_EXPONENT_1, expo1, &expo1_len);

	if ((expo1_len == 0) && (rv == CKR_OK)) {
		rv = soft_rsa_encrypt(key, in, in_len, out, 0);
		goto clean1;
	} else {
		if (rv != CKR_OK)
			goto clean1;
	}

	rv = soft_get_private_value(key, CKA_EXPONENT_2, expo2, &expo2_len);

	if ((expo2_len == 0) && (rv == CKR_OK)) {
		rv = soft_rsa_encrypt(key, in, in_len, out, 0);
		goto clean1;
	} else {
		if (rv != CKR_OK)
			goto clean1;
	}

	rv = soft_get_private_value(key, CKA_COEFFICIENT, coef, &coef_len);

	if ((coef_len == 0) && (rv == CKR_OK)) {
		rv = soft_rsa_encrypt(key, in, in_len, out, 0);
		goto clean1;
	} else {
		if (rv != CKR_OK)
			goto clean1;
	}

	k.modulus = modulus;
	k.modulus_bits = CRYPTO_BYTES2BITS(modulus_len);
	k.prime1 = prime1;
	k.prime1_bytes = prime1_len;
	k.prime2 = prime2;
	k.prime2_bytes = prime2_len;
	k.expo1 = expo1;
	k.expo1_bytes = expo1_len;
	k.expo2 = expo2;
	k.expo2_bytes = expo2_len;
	k.coeff = coef;
	k.coeff_bytes = coef_len;
	k.rfunc = NULL;

	rv = rsa_decrypt(&k, in, in_len, out);

clean1:

	return (rv);
}

/*
 * Allocate a RSA context for the active encryption or decryption operation.
 * This function is called without the session lock held.
 */
CK_RV
soft_rsa_crypt_init_common(soft_session_t *session_p,
    CK_MECHANISM_PTR pMechanism, soft_object_t *key_p,
    boolean_t encrypt)
{

	soft_rsa_ctx_t *rsa_ctx;
	soft_object_t *tmp_key = NULL;
	CK_RV rv;

	rsa_ctx = calloc(1, sizeof (soft_rsa_ctx_t));
	if (rsa_ctx == NULL) {
		return (CKR_HOST_MEMORY);
	}

	/*
	 * Make a copy of the encryption or decryption key, and save it
	 * in the RSA crypto context since it will be used later for
	 * encryption/decryption. We don't want to hold any object reference
	 * on this original key while doing encryption/decryption.
	 */
	(void) pthread_mutex_lock(&key_p->object_mutex);
	rv = soft_copy_object(key_p, &tmp_key, SOFT_COPY_OBJ_ORIG_SH,
	    NULL);

	if ((rv != CKR_OK) || (tmp_key == NULL)) {
		/* Most likely we ran out of space. */
		(void) pthread_mutex_unlock(&key_p->object_mutex);
		free(rsa_ctx);
		return (rv);
	}

	/* No need to hold the lock on the old object. */
	(void) pthread_mutex_unlock(&key_p->object_mutex);
	rsa_ctx->key = tmp_key;

	(void) pthread_mutex_lock(&session_p->session_mutex);
	if (encrypt) {
		/* Called by C_EncryptInit. */
		session_p->encrypt.context = rsa_ctx;
		session_p->encrypt.mech.mechanism = pMechanism->mechanism;
	} else {
		/* Called by C_DecryptInit. */
		session_p->decrypt.context = rsa_ctx;
		session_p->decrypt.mech.mechanism = pMechanism->mechanism;
	}
	(void) pthread_mutex_unlock(&session_p->session_mutex);

	return (CKR_OK);
}

CK_RV
soft_rsa_encrypt_common(soft_session_t *session_p, CK_BYTE_PTR pData,
    CK_ULONG ulDataLen, CK_BYTE_PTR pEncrypted,
    CK_ULONG_PTR pulEncryptedLen, CK_MECHANISM_TYPE mechanism)
{

	soft_rsa_ctx_t *rsa_ctx = session_p->encrypt.context;
	soft_object_t *key = rsa_ctx->key;
	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
	uint32_t modulus_len = sizeof (modulus);
	CK_BYTE	plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
	CK_BYTE	cipher_data[MAX_RSA_KEYLENGTH_IN_BYTES];
	CK_RV rv = CKR_OK;

	rv = soft_get_public_value(key, CKA_MODULUS, modulus, &modulus_len);
	if (rv != CKR_OK) {
		goto clean_exit;
	}

	if (pEncrypted == NULL) {
		/*
		 * Application asks for the length of the output buffer
		 * to hold the ciphertext.
		 */
		*pulEncryptedLen = modulus_len;
		rv = CKR_OK;
		goto clean1;
	}

	if (mechanism == CKM_RSA_PKCS) {
		/*
		 * Input data length needs to be <=
		 * modulus length-MIN_PKCS1_PADLEN.
		 */
		if (ulDataLen > ((CK_ULONG)modulus_len - MIN_PKCS1_PADLEN)) {
			*pulEncryptedLen = modulus_len;
			rv = CKR_DATA_LEN_RANGE;
			goto clean_exit;
		}
	} else {
		/* Input data length needs to be <= modulus length. */
		if (ulDataLen > (CK_ULONG)modulus_len) {
			*pulEncryptedLen = modulus_len;
			rv = CKR_DATA_LEN_RANGE;
			goto clean_exit;
		}
	}

	/* Is the application-supplied buffer large enough? */
	if (*pulEncryptedLen < (CK_ULONG)modulus_len) {
		*pulEncryptedLen = modulus_len;
		rv = CKR_BUFFER_TOO_SMALL;
		goto clean1;
	}

	if (mechanism == CKM_RSA_PKCS) {
		/*
		 * Add PKCS padding to the input data to format a block
		 * type "02" encryption block.
		 */
		rv = pkcs1_encode(PKCS1_ENCRYPT, pData, ulDataLen, plain_data,
		    modulus_len);

		if (rv != CKR_OK)
			goto clean_exit;
	} else {
		/* Pad zeros for the leading bytes of the input data. */
		(void) memset(plain_data, 0x0, modulus_len - ulDataLen);
		(void) memcpy(&plain_data[modulus_len - ulDataLen], pData,
		    ulDataLen);
	}

	rv = soft_rsa_encrypt(key, plain_data, modulus_len, cipher_data, 1);
	if (rv == CKR_OK) {
		(void) memcpy(pEncrypted, cipher_data, modulus_len);
		*pulEncryptedLen = modulus_len;
	}

clean_exit:
	(void) pthread_mutex_lock(&session_p->session_mutex);
	free(session_p->encrypt.context);
	session_p->encrypt.context = NULL;
	(void) pthread_mutex_unlock(&session_p->session_mutex);
	soft_cleanup_object(key);
	free(key);
clean1:
	return (rv);
}


CK_RV
soft_rsa_decrypt_common(soft_session_t *session_p, CK_BYTE_PTR pEncrypted,
    CK_ULONG ulEncryptedLen, CK_BYTE_PTR pData,
    CK_ULONG_PTR pulDataLen, CK_MECHANISM_TYPE mechanism)
{

	soft_rsa_ctx_t *rsa_ctx = session_p->decrypt.context;
	soft_object_t *key = rsa_ctx->key;
	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
	uint32_t modulus_len = sizeof (modulus);
	CK_BYTE	plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
	CK_RV rv = CKR_OK;

	rv = soft_get_private_value(key, CKA_MODULUS, modulus, &modulus_len);
	if (rv != CKR_OK) {
		goto clean_exit;
	}

	if (ulEncryptedLen != (CK_ULONG)modulus_len) {
		rv = CKR_ENCRYPTED_DATA_LEN_RANGE;
		goto clean_exit;
	}

	if (pData == NULL) {
		/*
		 * Application asks for the length of the output buffer
		 * to hold the recovered data.
		 */
		*pulDataLen = modulus_len;
		rv = CKR_OK;
		goto clean1;
	}

	if (mechanism == CKM_RSA_X_509) {
		if (*pulDataLen < (CK_ULONG)modulus_len) {
			*pulDataLen = modulus_len;
			rv = CKR_BUFFER_TOO_SMALL;
			goto clean1;
		}
	}

	rv = soft_rsa_decrypt(key, pEncrypted, modulus_len, plain_data);
	if (rv != CKR_OK) {
		goto clean_exit;
	}

	if (mechanism == CKM_RSA_PKCS) {
		size_t plain_len = modulus_len;
		size_t num_padding;

		/* Strip off the PKCS block formatting data. */
		rv = pkcs1_decode(PKCS1_DECRYPT, plain_data, &plain_len);
		if (rv != CKR_OK)
			goto clean_exit;

		num_padding = modulus_len - plain_len;
		if (ulEncryptedLen - num_padding > *pulDataLen) {
			*pulDataLen = plain_len;
			rv = CKR_BUFFER_TOO_SMALL;
			goto clean1;
		}

		(void) memcpy(pData, &plain_data[num_padding], plain_len);
		*pulDataLen = plain_len;
	} else {
		(void) memcpy(pData, plain_data, modulus_len);
		*pulDataLen = modulus_len;
	}

clean_exit:
	(void) pthread_mutex_lock(&session_p->session_mutex);
	free(session_p->decrypt.context);
	session_p->decrypt.context = NULL;
	(void) pthread_mutex_unlock(&session_p->session_mutex);
	soft_cleanup_object(key);
	free(key);

clean1:
	return (rv);
}

/*
 * Allocate a RSA context for the active sign or verify operation.
 * This function is called without the session lock held.
 */
CK_RV
soft_rsa_sign_verify_init_common(soft_session_t *session_p,
    CK_MECHANISM_PTR pMechanism, soft_object_t *key_p,
    boolean_t sign)
{
	CK_RV rv = CKR_OK;
	soft_rsa_ctx_t *rsa_ctx;
	CK_MECHANISM digest_mech;
	soft_object_t *tmp_key = NULL;

	if (sign) {
		if ((key_p->class != CKO_PRIVATE_KEY) ||
		    (key_p->key_type != CKK_RSA))
			return (CKR_KEY_TYPE_INCONSISTENT);
	} else {
		if ((key_p->class != CKO_PUBLIC_KEY) ||
		    (key_p->key_type != CKK_RSA))
			return (CKR_KEY_TYPE_INCONSISTENT);
	}

	switch (pMechanism->mechanism) {
	case CKM_MD5_RSA_PKCS:
		digest_mech.mechanism = CKM_MD5;
		rv = soft_digest_init_internal(session_p, &digest_mech);
		if (rv != CKR_OK)
			return (rv);
		break;

	case CKM_SHA1_RSA_PKCS:
		digest_mech.mechanism = CKM_SHA_1;
		digest_mech.pParameter = pMechanism->pParameter;
		digest_mech.ulParameterLen = pMechanism->ulParameterLen;
		rv = soft_digest_init_internal(session_p, &digest_mech);
		if (rv != CKR_OK)
			return (rv);
		break;

	case CKM_SHA256_RSA_PKCS:
		digest_mech.mechanism = CKM_SHA256;
		rv = soft_digest_init_internal(session_p, &digest_mech);
		if (rv != CKR_OK)
			return (rv);
		break;

	case CKM_SHA384_RSA_PKCS:
		digest_mech.mechanism = CKM_SHA384;
		rv = soft_digest_init_internal(session_p, &digest_mech);
		if (rv != CKR_OK)
			return (rv);
		break;

	case CKM_SHA512_RSA_PKCS:
		digest_mech.mechanism = CKM_SHA512;
		rv = soft_digest_init_internal(session_p, &digest_mech);
		if (rv != CKR_OK)
			return (rv);
		break;
	}

	rsa_ctx = malloc(sizeof (soft_rsa_ctx_t));

	if (rsa_ctx == NULL) {
		rv = CKR_HOST_MEMORY;
		goto clean_exit;
	}

	(void) pthread_mutex_lock(&key_p->object_mutex);
	rv = soft_copy_object(key_p, &tmp_key, SOFT_COPY_OBJ_ORIG_SH,
	    NULL);

	if ((rv != CKR_OK) || (tmp_key == NULL)) {
		/* Most likely we ran out of space. */
		(void) pthread_mutex_unlock(&key_p->object_mutex);
		free(rsa_ctx);
		goto clean_exit;
	}

	/* No need to hold the lock on the old object. */
	(void) pthread_mutex_unlock(&key_p->object_mutex);
	rsa_ctx->key = tmp_key;

	(void) pthread_mutex_lock(&session_p->session_mutex);

	if (sign) {
		session_p->sign.context = rsa_ctx;
		session_p->sign.mech.mechanism = pMechanism->mechanism;
	} else {
		session_p->verify.context = rsa_ctx;
		session_p->verify.mech.mechanism = pMechanism->mechanism;
	}

	(void) pthread_mutex_unlock(&session_p->session_mutex);

	return (CKR_OK);

clean_exit:
	(void) pthread_mutex_lock(&session_p->session_mutex);
	if (session_p->digest.context != NULL) {
		free(session_p->digest.context);
		session_p->digest.context = NULL;
		session_p->digest.flags = 0;
	}
	(void) pthread_mutex_unlock(&session_p->session_mutex);
	return (rv);

}


CK_RV
soft_rsa_sign_common(soft_session_t *session_p, CK_BYTE_PTR pData,
    CK_ULONG ulDataLen, CK_BYTE_PTR pSigned,
    CK_ULONG_PTR pulSignedLen, CK_MECHANISM_TYPE mechanism)
{

	CK_RV rv = CKR_OK;
	soft_rsa_ctx_t *rsa_ctx = session_p->sign.context;
	soft_object_t *key = rsa_ctx->key;
	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
	uint32_t modulus_len = sizeof (modulus);
	CK_BYTE	plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
	CK_BYTE	signed_data[MAX_RSA_KEYLENGTH_IN_BYTES];

	rv = soft_get_private_value(key, CKA_MODULUS, modulus, &modulus_len);
	if (rv != CKR_OK) {
		goto clean_exit;
	}

	if (pSigned == NULL) {
		/* Application asks for the length of the output buffer. */
		*pulSignedLen = modulus_len;
		rv = CKR_OK;
		goto clean1;
	}

	switch (mechanism) {

	case CKM_RSA_PKCS:

		/*
		 * Input data length needs to be <=
		 * modulus length-MIN_PKCS1_PADLEN.
		 */
		if (ulDataLen > ((CK_ULONG)modulus_len - MIN_PKCS1_PADLEN)) {
			*pulSignedLen = modulus_len;
			rv = CKR_DATA_LEN_RANGE;
			goto clean_exit;
		}
		break;

	case CKM_RSA_X_509:

		/* Input data length needs to be <= modulus length. */
		if (ulDataLen > (CK_ULONG)modulus_len) {
			*pulSignedLen = modulus_len;
			rv = CKR_DATA_LEN_RANGE;
			goto clean_exit;
		}
		break;
	}

	/* Is the application-supplied buffer large enough? */
	if (*pulSignedLen < (CK_ULONG)modulus_len) {
		*pulSignedLen = modulus_len;
		rv = CKR_BUFFER_TOO_SMALL;
		goto clean1;
	}

	switch (mechanism) {

	case CKM_RSA_PKCS:
	case CKM_MD5_RSA_PKCS:
	case CKM_SHA1_RSA_PKCS:
	case CKM_SHA256_RSA_PKCS:
	case CKM_SHA384_RSA_PKCS:
	case CKM_SHA512_RSA_PKCS:
		/*
		 * Add PKCS padding to the input data to format a block
		 * type "01" encryption block.
		 */
		rv = pkcs1_encode(PKCS1_SIGN, pData, ulDataLen, plain_data,
		    modulus_len);

		if (rv != CKR_OK) {
			goto clean_exit;
		}
		break;

	case CKM_RSA_X_509:

		/* Pad zeros for the leading bytes of the input data. */
		(void) memset(plain_data, 0x0, modulus_len - ulDataLen);
		(void) memcpy(&plain_data[modulus_len - ulDataLen], pData,
		    ulDataLen);
		break;
	}

	/*
	 * Perform RSA encryption with the signer's RSA private key
	 * for signature process.
	 */
	rv = soft_rsa_decrypt(key, plain_data, modulus_len, signed_data);

	if (rv == CKR_OK) {
		(void) memcpy(pSigned, signed_data, modulus_len);
		*pulSignedLen = modulus_len;
	}

clean_exit:
	(void) pthread_mutex_lock(&session_p->session_mutex);
	free(session_p->sign.context);
	session_p->sign.context = NULL;
	if (session_p->digest.context != NULL) {
		free(session_p->digest.context);
		session_p->digest.context = NULL;
		session_p->digest.flags = 0;
	}
	(void) pthread_mutex_unlock(&session_p->session_mutex);
	soft_cleanup_object(key);
	free(key);

clean1:
	return (rv);
}


CK_RV
soft_rsa_verify_common(soft_session_t *session_p, CK_BYTE_PTR pData,
    CK_ULONG ulDataLen, CK_BYTE_PTR pSignature,
    CK_ULONG ulSignatureLen, CK_MECHANISM_TYPE mechanism)
{

	CK_RV rv = CKR_OK;
	soft_rsa_ctx_t *rsa_ctx = session_p->verify.context;
	soft_object_t *key = rsa_ctx->key;
	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
	uint32_t modulus_len = sizeof (modulus);
	CK_BYTE	plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];

	rv = soft_get_public_value(key, CKA_MODULUS, modulus, &modulus_len);
	if (rv != CKR_OK) {
		goto clean_exit;
	}

	if (ulDataLen == 0) {
		rv = CKR_DATA_LEN_RANGE;
		goto clean_exit;
	}

	if (ulSignatureLen != (CK_ULONG)modulus_len) {
		rv = CKR_SIGNATURE_LEN_RANGE;
		goto clean_exit;
	}

	/*
	 * Perform RSA decryption with the signer's RSA public key
	 * for verification process.
	 */
	rv = soft_rsa_encrypt(key, pSignature, modulus_len, plain_data, 1);
	if (rv == CKR_OK) {
		switch (mechanism) {

		case CKM_RSA_PKCS:
		case CKM_MD5_RSA_PKCS:
		case CKM_SHA1_RSA_PKCS:
		case CKM_SHA256_RSA_PKCS:
		case CKM_SHA384_RSA_PKCS:
		case CKM_SHA512_RSA_PKCS:
		{
			/*
			 * Strip off the encoded padding bytes in front of the
			 * recovered data, then compare the recovered data with
			 * the original data.
			 */
			size_t data_len = modulus_len;

			rv = pkcs1_decode(PKCS1_VERIFY, plain_data, &data_len);
			if (rv != CKR_OK) {
				goto clean_exit;
			}

			if ((CK_ULONG)data_len != ulDataLen) {
				rv = CKR_DATA_LEN_RANGE;
				goto clean_exit;
			} else if (memcmp(pData,
			    &plain_data[modulus_len - data_len],
			    ulDataLen) != 0) {
				rv = CKR_SIGNATURE_INVALID;
				goto clean_exit;
			}
			break;
		}

		case CKM_RSA_X_509:
			/*
			 * Strip off the encoded padding bytes in front of the
			 * recovered plain_data, then compare the input data
			 * with the recovered data.
			 */
			if (memcmp(pData,
			    plain_data + ulSignatureLen - ulDataLen,
			    ulDataLen) != 0) {
				rv = CKR_SIGNATURE_INVALID;
				goto clean_exit;
			}
			break;
		}
	}

	if (rv == CKR_DATA_LEN_RANGE) {
		if ((mechanism == CKM_MD5_RSA_PKCS) ||
		    (mechanism == CKM_SHA1_RSA_PKCS) ||
		    (mechanism == CKM_SHA256_RSA_PKCS) ||
		    (mechanism == CKM_SHA384_RSA_PKCS) ||
		    (mechanism == CKM_SHA512_RSA_PKCS))
			rv = CKR_SIGNATURE_INVALID;
	}

clean_exit:
	(void) pthread_mutex_lock(&session_p->session_mutex);
	free(session_p->verify.context);
	session_p->verify.context = NULL;
	if (session_p->digest.context != NULL) {
		free(session_p->digest.context);
		session_p->digest.context = NULL;
		session_p->digest.flags = 0;
	}
	(void) pthread_mutex_unlock(&session_p->session_mutex);
	soft_cleanup_object(key);
	free(key);
	return (rv);
}

CK_RV
soft_genRSAkey_set_attribute(soft_object_t *key, CK_ATTRIBUTE_TYPE type,
    uchar_t *buf, uint32_t buflen, boolean_t public)
{
	CK_RV rv = CKR_OK;
	biginteger_t *dst = NULL;
	biginteger_t src;

	switch (type) {

	case CKA_MODULUS:

		if (public)
			dst = OBJ_PUB_RSA_MOD(key);
		else
			dst = OBJ_PRI_RSA_MOD(key);
		break;

	case CKA_PUBLIC_EXPONENT:

		if (public)
			dst = OBJ_PUB_RSA_PUBEXPO(key);
		else
			dst = OBJ_PRI_RSA_PUBEXPO(key);
		break;

	case CKA_PRIVATE_EXPONENT:

		dst = OBJ_PRI_RSA_PRIEXPO(key);
		break;

	case CKA_PRIME_1:

		dst = OBJ_PRI_RSA_PRIME1(key);
		break;

	case CKA_PRIME_2:

		dst = OBJ_PRI_RSA_PRIME2(key);
		break;

	case CKA_EXPONENT_1:

		dst = OBJ_PRI_RSA_EXPO1(key);
		break;

	case CKA_EXPONENT_2:

		dst = OBJ_PRI_RSA_EXPO2(key);
		break;

	case CKA_COEFFICIENT:

		dst = OBJ_PRI_RSA_COEF(key);
		break;
	}

	/* Note: no explanation found for why this is needed */
	while (buf[0] == 0) {	/* remove proceeding 0x00 */
		buf++;
		buflen--;
	}

	if ((rv = dup_bigint_attr(&src, buf, buflen)) != CKR_OK)
		goto cleanexit;

	/* Copy the attribute in the key object. */
	copy_bigint_attr(&src, dst);

cleanexit:
	return (rv);

}


CK_RV
soft_rsa_genkey_pair(soft_object_t *pubkey, soft_object_t *prikey)
{
	CK_RV rv = CKR_OK;
	CK_ATTRIBUTE template;
	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
	uint32_t modulus_len;
	uchar_t pub_expo[MAX_KEY_ATTR_BUFLEN];
	uint32_t pub_expo_len = sizeof (pub_expo);
	uchar_t private_exponent[MAX_KEY_ATTR_BUFLEN];
	uint32_t private_exponent_len = sizeof (private_exponent);
	uchar_t prime1[MAX_KEY_ATTR_BUFLEN];
	uint32_t prime1_len = sizeof (prime1);
	uchar_t prime2[MAX_KEY_ATTR_BUFLEN];
	uint32_t prime2_len = sizeof (prime2);
	uchar_t exponent1[MAX_KEY_ATTR_BUFLEN];
	uint32_t exponent1_len = sizeof (exponent1);
	uchar_t exponent2[MAX_KEY_ATTR_BUFLEN];
	uint32_t exponent2_len = sizeof (exponent2);
	uchar_t coefficient[MAX_KEY_ATTR_BUFLEN];
	uint32_t coefficient_len = sizeof (coefficient);
	RSAbytekey k;

	if ((pubkey == NULL) || (prikey == NULL)) {
		return (CKR_ARGUMENTS_BAD);
	}

	template.pValue = malloc(sizeof (CK_ULONG));
	if (template.pValue == NULL) {
		return (CKR_HOST_MEMORY);
	}
	template.ulValueLen = sizeof (CK_ULONG);

	rv = get_ulong_attr_from_object(OBJ_PUB_RSA_MOD_BITS(pubkey),
	    &template);
	if (rv != CKR_OK) {
		free(template.pValue);
		goto clean0;
	}

#ifdef	__sparcv9
	/* LINTED */
	modulus_len = (uint32_t)(*((CK_ULONG *)(template.pValue)));
#else	/* !__sparcv9 */
	modulus_len = *((CK_ULONG *)(template.pValue));
#endif	/* __sparcv9 */

	free(template.pValue);

	rv = soft_get_public_value(pubkey, CKA_PUBLIC_EXPONENT, pub_expo,
	    &pub_expo_len);
	if (rv != CKR_OK) {
		goto clean0;
	}

	/* Inputs to RSA key pair generation */
	k.modulus_bits = modulus_len;		/* save modulus len in bits  */
	modulus_len = CRYPTO_BITS2BYTES(modulus_len);	/* convert to bytes */
	k.modulus = modulus;
	k.pubexpo = pub_expo;
	k.pubexpo_bytes = pub_expo_len;
	k.rfunc = (IS_TOKEN_OBJECT(pubkey) || IS_TOKEN_OBJECT(prikey)) ?
	    pkcs11_get_random : pkcs11_get_urandom;

	/* Outputs from RSA key pair generation */
	k.privexpo = private_exponent;
	k.privexpo_bytes = private_exponent_len;
	k.prime1 = prime1;
	k.prime1_bytes = prime1_len;
	k.prime2 = prime2;
	k.prime2_bytes = prime2_len;
	k.expo1 = exponent1;
	k.expo1_bytes = exponent1_len;
	k.expo2 = exponent2;
	k.expo2_bytes = exponent2_len;
	k.coeff = coefficient;
	k.coeff_bytes = coefficient_len;

	rv = rsa_genkey_pair(&k);

	if (rv != CKR_OK) {
		goto clean0;
	}

	/*
	 * Add modulus in public template, and add all eight key fields
	 * in private template.
	 */
	if ((rv = soft_genRSAkey_set_attribute(pubkey, CKA_MODULUS,
	    modulus, CRYPTO_BITS2BYTES(k.modulus_bits), B_TRUE)) != CKR_OK) {
		goto clean0;
	}

	if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_MODULUS,
	    modulus, CRYPTO_BITS2BYTES(k.modulus_bits), B_FALSE)) != CKR_OK) {
		goto clean0;
	}

	if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_PRIVATE_EXPONENT,
	    private_exponent, k.privexpo_bytes, B_FALSE)) != CKR_OK) {
		goto clean0;
	}

	if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_PUBLIC_EXPONENT,
	    pub_expo, k.pubexpo_bytes, B_FALSE)) != CKR_OK) {
		goto clean0;
	}

	if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_PRIME_1,
	    prime1, k.prime1_bytes, B_FALSE)) != CKR_OK) {
		goto clean0;
	}

	if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_PRIME_2,
	    prime2, k.prime2_bytes, B_FALSE)) != CKR_OK) {
		goto clean0;
	}

	if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_EXPONENT_1,
	    exponent1, k.expo1_bytes, B_FALSE)) != CKR_OK) {
		goto clean0;
	}

	if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_EXPONENT_2,
	    exponent2, k.expo2_bytes, B_FALSE)) != CKR_OK) {
		goto clean0;
	}

	if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_COEFFICIENT,
	    coefficient, k.coeff_bytes, B_FALSE)) != CKR_OK) {
		goto clean0;
	}

clean0:
	return (rv);
}


CK_ULONG
get_rsa_sha1_prefix(CK_MECHANISM_PTR mech, CK_BYTE_PTR *prefix) {
	if (mech->pParameter == NULL) {
		*prefix = (CK_BYTE *)SHA1_DER_PREFIX;
		return (SHA1_DER_PREFIX_Len);
	}

	*prefix = (CK_BYTE *)SHA1_DER_PREFIX_OID;
	return (SHA1_DER_PREFIX_OID_Len);
}

CK_RV
soft_rsa_digest_sign_common(soft_session_t *session_p, CK_BYTE_PTR pData,
    CK_ULONG ulDataLen, CK_BYTE_PTR pSigned,
    CK_ULONG_PTR pulSignedLen, CK_MECHANISM_TYPE mechanism, boolean_t Final)
{

	CK_RV rv = CKR_OK;
	CK_BYTE hash[SHA512_DIGEST_LENGTH];  /* space enough for all mechs */
	CK_ULONG hash_len = SHA512_DIGEST_LENGTH;
	/* space enough for all mechs */
	CK_BYTE der_data[SHA512_DIGEST_LENGTH + SHA2_DER_PREFIX_Len];
	CK_ULONG der_data_len;
	soft_rsa_ctx_t *rsa_ctx = session_p->sign.context;
	soft_object_t *key = rsa_ctx->key;
	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
	uint32_t modulus_len = sizeof (modulus);
	CK_ULONG der_len;
	CK_BYTE_PTR der_prefix;

	rv = soft_get_private_value(key, CKA_MODULUS, modulus, &modulus_len);
	if (rv != CKR_OK) {
		(void) pthread_mutex_lock(&session_p->session_mutex);
		free(session_p->digest.context);
		session_p->digest.context = NULL;
		session_p->digest.flags = 0;
		(void) pthread_mutex_unlock(&session_p->session_mutex);
		soft_cleanup_object(key);
		free(key);
		goto clean1;
	}

	/* Check arguments before performing message digest. */
	if (pSigned == NULL) {
		/* Application asks for the length of the output buffer. */
		*pulSignedLen = modulus_len;
		rv = CKR_OK;
		goto clean1;
	}

	/* Is the application-supplied buffer large enough? */
	if (*pulSignedLen < (CK_ULONG)modulus_len) {
		*pulSignedLen = modulus_len;
		rv = CKR_BUFFER_TOO_SMALL;
		goto clean1;
	}

	if (Final) {
		rv = soft_digest_final(session_p, hash, &hash_len);
	} else {
		rv = soft_digest(session_p, pData, ulDataLen, hash, &hash_len);
	}

	if (rv != CKR_OK) {
		/* free the signature key */
		soft_cleanup_object(key);
		free(key);
		goto clean_exit;
	}

	/*
	 * Prepare the DER encoding of the DigestInfo value by setting it to:
	 *	<MECH>_DER_PREFIX || H
	 *
	 * See rsa_impl.c for more details.
	 */
	switch (session_p->digest.mech.mechanism) {
	case CKM_MD5:
		(void) memcpy(der_data, MD5_DER_PREFIX, MD5_DER_PREFIX_Len);
		(void) memcpy(der_data + MD5_DER_PREFIX_Len, hash, hash_len);
		der_data_len = MD5_DER_PREFIX_Len + hash_len;
		break;
	case CKM_SHA_1:
		der_len = get_rsa_sha1_prefix(&(session_p->digest.mech),
		    &der_prefix);
		(void) memcpy(der_data, der_prefix, der_len);
		(void) memcpy(der_data + der_len, hash, hash_len);
		der_data_len = der_len + hash_len;
		break;
	case CKM_SHA256:
		(void) memcpy(der_data, SHA256_DER_PREFIX,
		    SHA2_DER_PREFIX_Len);
		(void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
		der_data_len = SHA2_DER_PREFIX_Len + hash_len;
		break;
	case CKM_SHA384:
		(void) memcpy(der_data, SHA384_DER_PREFIX,
		    SHA2_DER_PREFIX_Len);
		(void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
		der_data_len = SHA2_DER_PREFIX_Len + hash_len;
		break;
	case CKM_SHA512:
		(void) memcpy(der_data, SHA512_DER_PREFIX,
		    SHA2_DER_PREFIX_Len);
		(void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
		der_data_len = SHA2_DER_PREFIX_Len + hash_len;
		break;
	}

	/*
	 * Now, we are ready to sign the DER_ENCODED data
	 * soft_rsa_sign_common() will free the signature key.
	 */
	rv = soft_rsa_sign_common(session_p, der_data, der_data_len,
	    pSigned, pulSignedLen, mechanism);

clean_exit:
	(void) pthread_mutex_lock(&session_p->session_mutex);
	/* soft_digest_common() has freed the digest context */
	session_p->digest.flags = 0;
	(void) pthread_mutex_unlock(&session_p->session_mutex);

clean1:
	return (rv);
}


CK_RV
soft_rsa_digest_verify_common(soft_session_t *session_p, CK_BYTE_PTR pData,
    CK_ULONG ulDataLen, CK_BYTE_PTR pSigned,
    CK_ULONG ulSignedLen, CK_MECHANISM_TYPE mechanism, boolean_t Final)
{

	CK_RV rv = CKR_OK;
	CK_BYTE hash[SHA512_DIGEST_LENGTH];  /* space for all mechs */
	CK_ULONG hash_len = SHA512_DIGEST_LENGTH;
	CK_BYTE der_data[SHA512_DIGEST_LENGTH + SHA2_DER_PREFIX_Len];
	CK_ULONG der_data_len;
	soft_rsa_ctx_t *rsa_ctx = session_p->verify.context;
	soft_object_t *key = rsa_ctx->key;
	CK_ULONG der_len;
	CK_BYTE_PTR der_prefix;

	if (Final) {
		rv = soft_digest_final(session_p, hash, &hash_len);
	} else {
		rv = soft_digest(session_p, pData, ulDataLen, hash, &hash_len);
	}

	if (rv != CKR_OK) {
		/* free the verification key */
		soft_cleanup_object(key);
		free(key);
		goto clean_exit;
	}

	/*
	 * Prepare the DER encoding of the DigestInfo value as follows:
	 * MD5:		MD5_DER_PREFIX || H
	 * SHA-1:	SHA1_DER_PREFIX || H
	 * SHA2:	SHA2_DER_PREFIX || H
	 *
	 * See rsa_impl.c for more details.
	 */
	switch (session_p->digest.mech.mechanism) {
	case CKM_MD5:
		(void) memcpy(der_data, MD5_DER_PREFIX, MD5_DER_PREFIX_Len);
		(void) memcpy(der_data + MD5_DER_PREFIX_Len, hash, hash_len);
		der_data_len = MD5_DER_PREFIX_Len + hash_len;
		break;
	case CKM_SHA_1:
		der_len = get_rsa_sha1_prefix(&(session_p->digest.mech),
		    &der_prefix);
		(void) memcpy(der_data, der_prefix, der_len);
		(void) memcpy(der_data + der_len, hash, hash_len);
		der_data_len = der_len + hash_len;
		break;
	case CKM_SHA256:
		(void) memcpy(der_data, SHA256_DER_PREFIX,
		    SHA2_DER_PREFIX_Len);
		(void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
		der_data_len = SHA2_DER_PREFIX_Len + hash_len;
		break;
	case CKM_SHA384:
		(void) memcpy(der_data, SHA384_DER_PREFIX,
		    SHA2_DER_PREFIX_Len);
		(void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
		der_data_len = SHA2_DER_PREFIX_Len + hash_len;
		break;
	case CKM_SHA512:
		(void) memcpy(der_data, SHA512_DER_PREFIX,
		    SHA2_DER_PREFIX_Len);
		(void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
		der_data_len = SHA2_DER_PREFIX_Len + hash_len;
		break;
	}

	/*
	 * Now, we are ready to verify the DER_ENCODED data using signature.
	 * soft_rsa_verify_common() will free the verification key.
	 */
	rv = soft_rsa_verify_common(session_p, der_data, der_data_len,
	    pSigned, ulSignedLen, mechanism);

clean_exit:
	(void) pthread_mutex_lock(&session_p->session_mutex);
	/* soft_digest_common() has freed the digest context */
	session_p->digest.flags = 0;
	(void) pthread_mutex_unlock(&session_p->session_mutex);

	return (rv);

}


CK_RV
soft_rsa_verify_recover(soft_session_t *session_p, CK_BYTE_PTR pSignature,
    CK_ULONG ulSignatureLen, CK_BYTE_PTR pData, CK_ULONG_PTR pulDataLen)
{

	CK_RV rv = CKR_OK;
	soft_rsa_ctx_t *rsa_ctx = session_p->verify.context;
	CK_MECHANISM_TYPE mechanism = session_p->verify.mech.mechanism;
	soft_object_t *key = rsa_ctx->key;
	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
	uint32_t modulus_len = sizeof (modulus);
	CK_BYTE	plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];

	rv = soft_get_public_value(key, CKA_MODULUS, modulus, &modulus_len);
	if (rv != CKR_OK) {
		goto clean_exit;
	}

	if (ulSignatureLen != (CK_ULONG)modulus_len) {
		rv = CKR_SIGNATURE_LEN_RANGE;
		goto clean_exit;
	}

	/*
	 * Perform RSA decryption with the signer's RSA public key
	 * for verification process.
	 */
	rv = soft_rsa_encrypt(key, pSignature, modulus_len, plain_data, 1);
	if (rv == CKR_OK) {
		switch (mechanism) {

		case CKM_RSA_PKCS:
		{
			/*
			 * Strip off the encoded padding bytes in front of the
			 * recovered data.
			 */
			size_t data_len = modulus_len;

			rv = pkcs1_decode(PKCS1_VERIFY, plain_data, &data_len);
			if (rv != CKR_OK) {
				goto clean_exit;
			}

			/*
			 * If application asks for the length of the output
			 * buffer?
			 */
			if (pData == NULL) {
				*pulDataLen = data_len;
				rv = CKR_OK;
				goto clean1;
			}

			/* Is the application-supplied buffer large enough? */
			if (*pulDataLen < (CK_ULONG)data_len) {
				*pulDataLen = data_len;
				rv = CKR_BUFFER_TOO_SMALL;
				goto clean1;
			}

			(void) memcpy(pData,
			    &plain_data[modulus_len - data_len], data_len);
			*pulDataLen = data_len;

			break;
		}

		case CKM_RSA_X_509:
			/*
			 * If application asks for the length of the output
			 * buffer?
			 */
			if (pData == NULL) {
				*pulDataLen = modulus_len;
				rv = CKR_OK;
				goto clean1;
			}

			/* Is the application-supplied buffer large enough? */
			if (*pulDataLen < (CK_ULONG)modulus_len) {
				*pulDataLen = modulus_len;
				rv = CKR_BUFFER_TOO_SMALL;
				goto clean1;
			}

			(void) memcpy(pData, plain_data, modulus_len);
			*pulDataLen = modulus_len;

			break;
		}
	}

clean_exit:
	(void) pthread_mutex_lock(&session_p->session_mutex);
	free(session_p->verify.context);
	session_p->verify.context = NULL;
	(void) pthread_mutex_unlock(&session_p->session_mutex);
	soft_cleanup_object(key);
	free(key);

clean1:
	return (rv);
}