1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* Asymmetric algorithms supported by virtio crypto device 3 * 4 * Authors: zhenwei pi <pizhenwei@bytedance.com> 5 * lei he <helei.sig11@bytedance.com> 6 * 7 * Copyright 2022 Bytedance CO., LTD. 8 */ 9 10 #include <crypto/engine.h> 11 #include <crypto/internal/akcipher.h> 12 #include <crypto/internal/rsa.h> 13 #include <crypto/scatterwalk.h> 14 #include <linux/err.h> 15 #include <linux/kernel.h> 16 #include <linux/mpi.h> 17 #include <linux/scatterlist.h> 18 #include <linux/slab.h> 19 #include <linux/string.h> 20 #include <uapi/linux/virtio_crypto.h> 21 #include "virtio_crypto_common.h" 22 23 struct virtio_crypto_rsa_ctx { 24 unsigned int key_size; 25 }; 26 27 struct virtio_crypto_akcipher_ctx { 28 struct virtio_crypto *vcrypto; 29 bool session_valid; 30 __u64 session_id; 31 union { 32 struct virtio_crypto_rsa_ctx rsa_ctx; 33 }; 34 }; 35 36 struct virtio_crypto_akcipher_request { 37 struct virtio_crypto_request base; 38 void *src_buf; 39 void *dst_buf; 40 uint32_t opcode; 41 }; 42 43 struct virtio_crypto_akcipher_algo { 44 uint32_t algonum; 45 uint32_t service; 46 unsigned int active_devs; 47 struct akcipher_engine_alg algo; 48 }; 49 50 static DEFINE_MUTEX(algs_lock); 51 52 static void virtio_crypto_akcipher_finalize_req( 53 struct virtio_crypto_akcipher_request *vc_akcipher_req, 54 struct akcipher_request *req, int err) 55 { 56 kfree(vc_akcipher_req->src_buf); 57 kfree(vc_akcipher_req->dst_buf); 58 vc_akcipher_req->src_buf = NULL; 59 vc_akcipher_req->dst_buf = NULL; 60 virtcrypto_clear_request(&vc_akcipher_req->base); 61 62 crypto_finalize_akcipher_request(vc_akcipher_req->base.dataq->engine, req, err); 63 } 64 65 static void virtio_crypto_dataq_akcipher_callback(struct virtio_crypto_request *vc_req, int len) 66 { 67 struct virtio_crypto_akcipher_request *vc_akcipher_req = 68 container_of(vc_req, struct virtio_crypto_akcipher_request, base); 69 struct akcipher_request *akcipher_req = 70 container_of((void *)vc_akcipher_req, struct akcipher_request, 71 __ctx); 72 int error; 73 74 switch (vc_req->status) { 75 case VIRTIO_CRYPTO_OK: 76 error = 0; 77 break; 78 case VIRTIO_CRYPTO_INVSESS: 79 case VIRTIO_CRYPTO_ERR: 80 error = -EINVAL; 81 break; 82 case VIRTIO_CRYPTO_BADMSG: 83 error = -EBADMSG; 84 break; 85 default: 86 error = -EIO; 87 break; 88 } 89 90 /* actual length may be less than dst buffer */ 91 akcipher_req->dst_len = min_t(unsigned int, len - sizeof(vc_req->status), 92 akcipher_req->dst_len); 93 sg_copy_from_buffer(akcipher_req->dst, sg_nents(akcipher_req->dst), 94 vc_akcipher_req->dst_buf, akcipher_req->dst_len); 95 virtio_crypto_akcipher_finalize_req(vc_akcipher_req, akcipher_req, error); 96 } 97 98 static int virtio_crypto_alg_akcipher_init_session(struct virtio_crypto_akcipher_ctx *ctx, 99 struct virtio_crypto_ctrl_header *header, 100 struct virtio_crypto_akcipher_session_para *para, 101 const uint8_t *key, unsigned int keylen) 102 { 103 struct scatterlist outhdr_sg, key_sg, inhdr_sg, *sgs[3]; 104 struct virtio_crypto *vcrypto = ctx->vcrypto; 105 uint8_t *pkey; 106 int err; 107 unsigned int num_out = 0, num_in = 0; 108 struct virtio_crypto_op_ctrl_req *ctrl; 109 struct virtio_crypto_session_input *input; 110 struct virtio_crypto_ctrl_request *vc_ctrl_req; 111 112 pkey = kmemdup(key, keylen, GFP_KERNEL); 113 if (!pkey) 114 return -ENOMEM; 115 116 vc_ctrl_req = kzalloc_obj(*vc_ctrl_req); 117 if (!vc_ctrl_req) { 118 err = -ENOMEM; 119 goto out; 120 } 121 122 ctrl = &vc_ctrl_req->ctrl; 123 memcpy(&ctrl->header, header, sizeof(ctrl->header)); 124 memcpy(&ctrl->u.akcipher_create_session.para, para, sizeof(*para)); 125 input = &vc_ctrl_req->input; 126 input->status = cpu_to_le32(VIRTIO_CRYPTO_ERR); 127 128 sg_init_one(&outhdr_sg, ctrl, sizeof(*ctrl)); 129 sgs[num_out++] = &outhdr_sg; 130 131 sg_init_one(&key_sg, pkey, keylen); 132 sgs[num_out++] = &key_sg; 133 134 sg_init_one(&inhdr_sg, input, sizeof(*input)); 135 sgs[num_out + num_in++] = &inhdr_sg; 136 137 err = virtio_crypto_ctrl_vq_request(vcrypto, sgs, num_out, num_in, vc_ctrl_req); 138 if (err < 0) 139 goto out; 140 141 if (le32_to_cpu(input->status) != VIRTIO_CRYPTO_OK) { 142 pr_err("virtio_crypto: Create session failed status: %u\n", 143 le32_to_cpu(input->status)); 144 err = -EINVAL; 145 goto out; 146 } 147 148 ctx->session_id = le64_to_cpu(input->session_id); 149 ctx->session_valid = true; 150 err = 0; 151 152 out: 153 kfree(vc_ctrl_req); 154 kfree_sensitive(pkey); 155 156 return err; 157 } 158 159 static int virtio_crypto_alg_akcipher_close_session(struct virtio_crypto_akcipher_ctx *ctx) 160 { 161 struct scatterlist outhdr_sg, inhdr_sg, *sgs[2]; 162 struct virtio_crypto_destroy_session_req *destroy_session; 163 struct virtio_crypto *vcrypto = ctx->vcrypto; 164 unsigned int num_out = 0, num_in = 0; 165 int err; 166 struct virtio_crypto_op_ctrl_req *ctrl; 167 struct virtio_crypto_inhdr *ctrl_status; 168 struct virtio_crypto_ctrl_request *vc_ctrl_req; 169 170 if (!ctx->session_valid) 171 return 0; 172 173 vc_ctrl_req = kzalloc_obj(*vc_ctrl_req); 174 if (!vc_ctrl_req) 175 return -ENOMEM; 176 177 ctrl_status = &vc_ctrl_req->ctrl_status; 178 ctrl_status->status = VIRTIO_CRYPTO_ERR; 179 ctrl = &vc_ctrl_req->ctrl; 180 ctrl->header.opcode = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_DESTROY_SESSION); 181 ctrl->header.queue_id = 0; 182 183 destroy_session = &ctrl->u.destroy_session; 184 destroy_session->session_id = cpu_to_le64(ctx->session_id); 185 186 sg_init_one(&outhdr_sg, ctrl, sizeof(*ctrl)); 187 sgs[num_out++] = &outhdr_sg; 188 189 sg_init_one(&inhdr_sg, &ctrl_status->status, sizeof(ctrl_status->status)); 190 sgs[num_out + num_in++] = &inhdr_sg; 191 192 err = virtio_crypto_ctrl_vq_request(vcrypto, sgs, num_out, num_in, vc_ctrl_req); 193 if (err < 0) 194 goto out; 195 196 if (ctrl_status->status != VIRTIO_CRYPTO_OK) { 197 pr_err("virtio_crypto: Close session failed status: %u, session_id: 0x%llx\n", 198 ctrl_status->status, 199 le64_to_cpu(destroy_session->session_id)); 200 err = -EINVAL; 201 goto out; 202 } 203 204 err = 0; 205 ctx->session_valid = false; 206 207 out: 208 kfree(vc_ctrl_req); 209 210 return err; 211 } 212 213 static int __virtio_crypto_akcipher_do_req(struct virtio_crypto_akcipher_request *vc_akcipher_req, 214 struct akcipher_request *req, struct data_queue *data_vq) 215 { 216 struct crypto_akcipher *atfm = crypto_akcipher_reqtfm(req); 217 struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(atfm); 218 struct virtio_crypto_request *vc_req = &vc_akcipher_req->base; 219 struct virtio_crypto *vcrypto = ctx->vcrypto; 220 struct virtio_crypto_op_data_req *req_data = vc_req->req_data; 221 struct scatterlist *sgs[4], outhdr_sg, inhdr_sg, srcdata_sg, dstdata_sg; 222 void *src_buf, *dst_buf = NULL; 223 unsigned int num_out = 0, num_in = 0; 224 int node = dev_to_node(&vcrypto->vdev->dev); 225 unsigned long flags; 226 int ret; 227 228 /* out header */ 229 sg_init_one(&outhdr_sg, req_data, sizeof(*req_data)); 230 sgs[num_out++] = &outhdr_sg; 231 232 /* src data */ 233 src_buf = kcalloc_node(req->src_len, 1, GFP_KERNEL, node); 234 if (!src_buf) 235 return -ENOMEM; 236 237 sg_copy_to_buffer(req->src, sg_nents(req->src), src_buf, req->src_len); 238 sg_init_one(&srcdata_sg, src_buf, req->src_len); 239 sgs[num_out++] = &srcdata_sg; 240 241 /* dst data */ 242 dst_buf = kcalloc_node(req->dst_len, 1, GFP_KERNEL, node); 243 if (!dst_buf) 244 goto free_src; 245 246 sg_init_one(&dstdata_sg, dst_buf, req->dst_len); 247 sgs[num_out + num_in++] = &dstdata_sg; 248 249 vc_akcipher_req->src_buf = src_buf; 250 vc_akcipher_req->dst_buf = dst_buf; 251 252 /* in header */ 253 sg_init_one(&inhdr_sg, &vc_req->status, sizeof(vc_req->status)); 254 sgs[num_out + num_in++] = &inhdr_sg; 255 256 spin_lock_irqsave(&data_vq->lock, flags); 257 ret = virtqueue_add_sgs(data_vq->vq, sgs, num_out, num_in, vc_req, GFP_ATOMIC); 258 virtqueue_kick(data_vq->vq); 259 spin_unlock_irqrestore(&data_vq->lock, flags); 260 if (ret) 261 goto err; 262 263 return 0; 264 265 err: 266 kfree(dst_buf); 267 free_src: 268 kfree(src_buf); 269 return -ENOMEM; 270 } 271 272 static int virtio_crypto_rsa_do_req(struct crypto_engine *engine, void *vreq) 273 { 274 struct akcipher_request *req = container_of(vreq, struct akcipher_request, base); 275 struct virtio_crypto_akcipher_request *vc_akcipher_req = akcipher_request_ctx(req); 276 struct virtio_crypto_request *vc_req = &vc_akcipher_req->base; 277 struct crypto_akcipher *atfm = crypto_akcipher_reqtfm(req); 278 struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(atfm); 279 struct virtio_crypto *vcrypto = ctx->vcrypto; 280 struct data_queue *data_vq = vc_req->dataq; 281 struct virtio_crypto_op_header *header; 282 struct virtio_crypto_akcipher_data_req *akcipher_req; 283 int ret; 284 285 vc_req->sgs = NULL; 286 vc_req->req_data = kzalloc_node(sizeof(*vc_req->req_data), 287 GFP_KERNEL, dev_to_node(&vcrypto->vdev->dev)); 288 if (!vc_req->req_data) 289 return -ENOMEM; 290 291 /* build request header */ 292 header = &vc_req->req_data->header; 293 header->opcode = cpu_to_le32(vc_akcipher_req->opcode); 294 header->algo = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_RSA); 295 header->session_id = cpu_to_le64(ctx->session_id); 296 297 /* build request akcipher data */ 298 akcipher_req = &vc_req->req_data->u.akcipher_req; 299 akcipher_req->para.src_data_len = cpu_to_le32(req->src_len); 300 akcipher_req->para.dst_data_len = cpu_to_le32(req->dst_len); 301 302 ret = __virtio_crypto_akcipher_do_req(vc_akcipher_req, req, data_vq); 303 if (ret < 0) { 304 kfree_sensitive(vc_req->req_data); 305 vc_req->req_data = NULL; 306 return ret; 307 } 308 309 return 0; 310 } 311 312 static int virtio_crypto_rsa_req(struct akcipher_request *req, uint32_t opcode) 313 { 314 struct crypto_akcipher *atfm = crypto_akcipher_reqtfm(req); 315 struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(atfm); 316 struct virtio_crypto_akcipher_request *vc_akcipher_req = akcipher_request_ctx(req); 317 struct virtio_crypto_request *vc_req = &vc_akcipher_req->base; 318 struct virtio_crypto *vcrypto = ctx->vcrypto; 319 /* Use the first data virtqueue as default */ 320 struct data_queue *data_vq = &vcrypto->data_vq[0]; 321 322 vc_req->dataq = data_vq; 323 vc_req->alg_cb = virtio_crypto_dataq_akcipher_callback; 324 vc_akcipher_req->opcode = opcode; 325 326 return crypto_transfer_akcipher_request_to_engine(data_vq->engine, req); 327 } 328 329 static int virtio_crypto_rsa_encrypt(struct akcipher_request *req) 330 { 331 return virtio_crypto_rsa_req(req, VIRTIO_CRYPTO_AKCIPHER_ENCRYPT); 332 } 333 334 static int virtio_crypto_rsa_decrypt(struct akcipher_request *req) 335 { 336 return virtio_crypto_rsa_req(req, VIRTIO_CRYPTO_AKCIPHER_DECRYPT); 337 } 338 339 static int virtio_crypto_rsa_set_key(struct crypto_akcipher *tfm, 340 const void *key, 341 unsigned int keylen, 342 bool private, 343 int padding_algo, 344 int hash_algo) 345 { 346 struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm); 347 struct virtio_crypto_rsa_ctx *rsa_ctx = &ctx->rsa_ctx; 348 struct virtio_crypto *vcrypto; 349 struct virtio_crypto_ctrl_header header; 350 struct virtio_crypto_akcipher_session_para para; 351 struct rsa_key rsa_key = {0}; 352 int node = virtio_crypto_get_current_node(); 353 uint32_t keytype; 354 int ret; 355 MPI n; 356 357 if (private) { 358 keytype = VIRTIO_CRYPTO_AKCIPHER_KEY_TYPE_PRIVATE; 359 ret = rsa_parse_priv_key(&rsa_key, key, keylen); 360 } else { 361 keytype = VIRTIO_CRYPTO_AKCIPHER_KEY_TYPE_PUBLIC; 362 ret = rsa_parse_pub_key(&rsa_key, key, keylen); 363 } 364 365 if (ret) 366 return ret; 367 368 n = mpi_read_raw_data(rsa_key.n, rsa_key.n_sz); 369 if (!n) 370 return -ENOMEM; 371 372 rsa_ctx->key_size = mpi_get_size(n); 373 mpi_free(n); 374 375 if (!ctx->vcrypto) { 376 vcrypto = virtcrypto_get_dev_node(node, VIRTIO_CRYPTO_SERVICE_AKCIPHER, 377 VIRTIO_CRYPTO_AKCIPHER_RSA); 378 if (!vcrypto) { 379 pr_err("virtio_crypto: Could not find a virtio device in the system or unsupported algo\n"); 380 return -ENODEV; 381 } 382 383 ctx->vcrypto = vcrypto; 384 } else { 385 virtio_crypto_alg_akcipher_close_session(ctx); 386 } 387 388 /* set ctrl header */ 389 header.opcode = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_CREATE_SESSION); 390 header.algo = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_RSA); 391 header.queue_id = 0; 392 393 /* set RSA para */ 394 para.algo = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_RSA); 395 para.keytype = cpu_to_le32(keytype); 396 para.keylen = cpu_to_le32(keylen); 397 para.u.rsa.padding_algo = cpu_to_le32(padding_algo); 398 para.u.rsa.hash_algo = cpu_to_le32(hash_algo); 399 400 return virtio_crypto_alg_akcipher_init_session(ctx, &header, ¶, key, keylen); 401 } 402 403 static int virtio_crypto_rsa_raw_set_priv_key(struct crypto_akcipher *tfm, 404 const void *key, 405 unsigned int keylen) 406 { 407 return virtio_crypto_rsa_set_key(tfm, key, keylen, 1, 408 VIRTIO_CRYPTO_RSA_RAW_PADDING, 409 VIRTIO_CRYPTO_RSA_NO_HASH); 410 } 411 412 413 static int virtio_crypto_p1pad_rsa_sha1_set_priv_key(struct crypto_akcipher *tfm, 414 const void *key, 415 unsigned int keylen) 416 { 417 return virtio_crypto_rsa_set_key(tfm, key, keylen, 1, 418 VIRTIO_CRYPTO_RSA_PKCS1_PADDING, 419 VIRTIO_CRYPTO_RSA_SHA1); 420 } 421 422 static int virtio_crypto_rsa_raw_set_pub_key(struct crypto_akcipher *tfm, 423 const void *key, 424 unsigned int keylen) 425 { 426 return virtio_crypto_rsa_set_key(tfm, key, keylen, 0, 427 VIRTIO_CRYPTO_RSA_RAW_PADDING, 428 VIRTIO_CRYPTO_RSA_NO_HASH); 429 } 430 431 static int virtio_crypto_p1pad_rsa_sha1_set_pub_key(struct crypto_akcipher *tfm, 432 const void *key, 433 unsigned int keylen) 434 { 435 return virtio_crypto_rsa_set_key(tfm, key, keylen, 0, 436 VIRTIO_CRYPTO_RSA_PKCS1_PADDING, 437 VIRTIO_CRYPTO_RSA_SHA1); 438 } 439 440 static unsigned int virtio_crypto_rsa_max_size(struct crypto_akcipher *tfm) 441 { 442 struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm); 443 struct virtio_crypto_rsa_ctx *rsa_ctx = &ctx->rsa_ctx; 444 445 return rsa_ctx->key_size; 446 } 447 448 static int virtio_crypto_rsa_init_tfm(struct crypto_akcipher *tfm) 449 { 450 akcipher_set_reqsize(tfm, 451 sizeof(struct virtio_crypto_akcipher_request)); 452 453 return 0; 454 } 455 456 static void virtio_crypto_rsa_exit_tfm(struct crypto_akcipher *tfm) 457 { 458 struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm); 459 460 virtio_crypto_alg_akcipher_close_session(ctx); 461 virtcrypto_dev_put(ctx->vcrypto); 462 } 463 464 static struct virtio_crypto_akcipher_algo virtio_crypto_akcipher_algs[] = { 465 { 466 .algonum = VIRTIO_CRYPTO_AKCIPHER_RSA, 467 .service = VIRTIO_CRYPTO_SERVICE_AKCIPHER, 468 .algo.base = { 469 .encrypt = virtio_crypto_rsa_encrypt, 470 .decrypt = virtio_crypto_rsa_decrypt, 471 .set_pub_key = virtio_crypto_rsa_raw_set_pub_key, 472 .set_priv_key = virtio_crypto_rsa_raw_set_priv_key, 473 .max_size = virtio_crypto_rsa_max_size, 474 .init = virtio_crypto_rsa_init_tfm, 475 .exit = virtio_crypto_rsa_exit_tfm, 476 .base = { 477 .cra_name = "rsa", 478 .cra_driver_name = "virtio-crypto-rsa", 479 .cra_priority = 150, 480 .cra_module = THIS_MODULE, 481 .cra_ctxsize = sizeof(struct virtio_crypto_akcipher_ctx), 482 }, 483 }, 484 .algo.op = { 485 .do_one_request = virtio_crypto_rsa_do_req, 486 }, 487 }, 488 { 489 .algonum = VIRTIO_CRYPTO_AKCIPHER_RSA, 490 .service = VIRTIO_CRYPTO_SERVICE_AKCIPHER, 491 .algo.base = { 492 .encrypt = virtio_crypto_rsa_encrypt, 493 .decrypt = virtio_crypto_rsa_decrypt, 494 /* 495 * Must specify an arbitrary hash algorithm upon 496 * set_{pub,priv}_key (even though it's not used 497 * by encrypt/decrypt) because qemu checks for it. 498 */ 499 .set_pub_key = virtio_crypto_p1pad_rsa_sha1_set_pub_key, 500 .set_priv_key = virtio_crypto_p1pad_rsa_sha1_set_priv_key, 501 .max_size = virtio_crypto_rsa_max_size, 502 .init = virtio_crypto_rsa_init_tfm, 503 .exit = virtio_crypto_rsa_exit_tfm, 504 .base = { 505 .cra_name = "pkcs1pad(rsa)", 506 .cra_driver_name = "virtio-pkcs1-rsa", 507 .cra_priority = 150, 508 .cra_module = THIS_MODULE, 509 .cra_ctxsize = sizeof(struct virtio_crypto_akcipher_ctx), 510 }, 511 }, 512 .algo.op = { 513 .do_one_request = virtio_crypto_rsa_do_req, 514 }, 515 }, 516 }; 517 518 int virtio_crypto_akcipher_algs_register(struct virtio_crypto *vcrypto) 519 { 520 int ret = 0; 521 int i = 0; 522 523 mutex_lock(&algs_lock); 524 525 for (i = 0; i < ARRAY_SIZE(virtio_crypto_akcipher_algs); i++) { 526 uint32_t service = virtio_crypto_akcipher_algs[i].service; 527 uint32_t algonum = virtio_crypto_akcipher_algs[i].algonum; 528 529 if (!virtcrypto_algo_is_supported(vcrypto, service, algonum)) 530 continue; 531 532 if (virtio_crypto_akcipher_algs[i].active_devs == 0) { 533 ret = crypto_engine_register_akcipher(&virtio_crypto_akcipher_algs[i].algo); 534 if (ret) 535 goto unlock; 536 } 537 538 virtio_crypto_akcipher_algs[i].active_devs++; 539 dev_info(&vcrypto->vdev->dev, "Registered akcipher algo %s\n", 540 virtio_crypto_akcipher_algs[i].algo.base.base.cra_name); 541 } 542 543 unlock: 544 mutex_unlock(&algs_lock); 545 return ret; 546 } 547 548 void virtio_crypto_akcipher_algs_unregister(struct virtio_crypto *vcrypto) 549 { 550 int i = 0; 551 552 mutex_lock(&algs_lock); 553 554 for (i = 0; i < ARRAY_SIZE(virtio_crypto_akcipher_algs); i++) { 555 uint32_t service = virtio_crypto_akcipher_algs[i].service; 556 uint32_t algonum = virtio_crypto_akcipher_algs[i].algonum; 557 558 if (virtio_crypto_akcipher_algs[i].active_devs == 0 || 559 !virtcrypto_algo_is_supported(vcrypto, service, algonum)) 560 continue; 561 562 if (virtio_crypto_akcipher_algs[i].active_devs == 1) 563 crypto_engine_unregister_akcipher(&virtio_crypto_akcipher_algs[i].algo); 564 565 virtio_crypto_akcipher_algs[i].active_devs--; 566 } 567 568 mutex_unlock(&algs_lock); 569 } 570