1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2020 Hannes Reinecke, SUSE Linux 4 */ 5 6 #include <linux/crc32.h> 7 #include <linux/base64.h> 8 #include <linux/prandom.h> 9 #include <asm/unaligned.h> 10 #include <crypto/hash.h> 11 #include <crypto/dh.h> 12 #include "nvme.h" 13 #include "fabrics.h" 14 #include <linux/nvme-auth.h> 15 16 #define CHAP_BUF_SIZE 4096 17 static struct kmem_cache *nvme_chap_buf_cache; 18 static mempool_t *nvme_chap_buf_pool; 19 20 struct nvme_dhchap_queue_context { 21 struct list_head entry; 22 struct work_struct auth_work; 23 struct nvme_ctrl *ctrl; 24 struct crypto_shash *shash_tfm; 25 struct crypto_kpp *dh_tfm; 26 struct nvme_dhchap_key *transformed_key; 27 void *buf; 28 int qid; 29 int error; 30 u32 s1; 31 u32 s2; 32 bool bi_directional; 33 u16 transaction; 34 u8 status; 35 u8 dhgroup_id; 36 u8 hash_id; 37 size_t hash_len; 38 u8 c1[64]; 39 u8 c2[64]; 40 u8 response[64]; 41 u8 *ctrl_key; 42 u8 *host_key; 43 u8 *sess_key; 44 int ctrl_key_len; 45 int host_key_len; 46 int sess_key_len; 47 }; 48 49 static struct workqueue_struct *nvme_auth_wq; 50 51 #define nvme_auth_flags_from_qid(qid) \ 52 (qid == 0) ? 0 : BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_RESERVED 53 #define nvme_auth_queue_from_qid(ctrl, qid) \ 54 (qid == 0) ? (ctrl)->fabrics_q : (ctrl)->connect_q 55 56 static inline int ctrl_max_dhchaps(struct nvme_ctrl *ctrl) 57 { 58 return ctrl->opts->nr_io_queues + ctrl->opts->nr_write_queues + 59 ctrl->opts->nr_poll_queues + 1; 60 } 61 62 static int nvme_auth_submit(struct nvme_ctrl *ctrl, int qid, 63 void *data, size_t data_len, bool auth_send) 64 { 65 struct nvme_command cmd = {}; 66 blk_mq_req_flags_t flags = nvme_auth_flags_from_qid(qid); 67 struct request_queue *q = nvme_auth_queue_from_qid(ctrl, qid); 68 int ret; 69 70 cmd.auth_common.opcode = nvme_fabrics_command; 71 cmd.auth_common.secp = NVME_AUTH_DHCHAP_PROTOCOL_IDENTIFIER; 72 cmd.auth_common.spsp0 = 0x01; 73 cmd.auth_common.spsp1 = 0x01; 74 if (auth_send) { 75 cmd.auth_send.fctype = nvme_fabrics_type_auth_send; 76 cmd.auth_send.tl = cpu_to_le32(data_len); 77 } else { 78 cmd.auth_receive.fctype = nvme_fabrics_type_auth_receive; 79 cmd.auth_receive.al = cpu_to_le32(data_len); 80 } 81 82 ret = __nvme_submit_sync_cmd(q, &cmd, NULL, data, data_len, 83 qid == 0 ? NVME_QID_ANY : qid, 84 0, flags); 85 if (ret > 0) 86 dev_warn(ctrl->device, 87 "qid %d auth_send failed with status %d\n", qid, ret); 88 else if (ret < 0) 89 dev_err(ctrl->device, 90 "qid %d auth_send failed with error %d\n", qid, ret); 91 return ret; 92 } 93 94 static int nvme_auth_receive_validate(struct nvme_ctrl *ctrl, int qid, 95 struct nvmf_auth_dhchap_failure_data *data, 96 u16 transaction, u8 expected_msg) 97 { 98 dev_dbg(ctrl->device, "%s: qid %d auth_type %d auth_id %x\n", 99 __func__, qid, data->auth_type, data->auth_id); 100 101 if (data->auth_type == NVME_AUTH_COMMON_MESSAGES && 102 data->auth_id == NVME_AUTH_DHCHAP_MESSAGE_FAILURE1) { 103 return data->rescode_exp; 104 } 105 if (data->auth_type != NVME_AUTH_DHCHAP_MESSAGES || 106 data->auth_id != expected_msg) { 107 dev_warn(ctrl->device, 108 "qid %d invalid message %02x/%02x\n", 109 qid, data->auth_type, data->auth_id); 110 return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE; 111 } 112 if (le16_to_cpu(data->t_id) != transaction) { 113 dev_warn(ctrl->device, 114 "qid %d invalid transaction ID %d\n", 115 qid, le16_to_cpu(data->t_id)); 116 return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE; 117 } 118 return 0; 119 } 120 121 static int nvme_auth_set_dhchap_negotiate_data(struct nvme_ctrl *ctrl, 122 struct nvme_dhchap_queue_context *chap) 123 { 124 struct nvmf_auth_dhchap_negotiate_data *data = chap->buf; 125 size_t size = sizeof(*data) + sizeof(union nvmf_auth_protocol); 126 127 if (size > CHAP_BUF_SIZE) { 128 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 129 return -EINVAL; 130 } 131 memset((u8 *)chap->buf, 0, size); 132 data->auth_type = NVME_AUTH_COMMON_MESSAGES; 133 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE; 134 data->t_id = cpu_to_le16(chap->transaction); 135 data->sc_c = 0; /* No secure channel concatenation */ 136 data->napd = 1; 137 data->auth_protocol[0].dhchap.authid = NVME_AUTH_DHCHAP_AUTH_ID; 138 data->auth_protocol[0].dhchap.halen = 3; 139 data->auth_protocol[0].dhchap.dhlen = 6; 140 data->auth_protocol[0].dhchap.idlist[0] = NVME_AUTH_HASH_SHA256; 141 data->auth_protocol[0].dhchap.idlist[1] = NVME_AUTH_HASH_SHA384; 142 data->auth_protocol[0].dhchap.idlist[2] = NVME_AUTH_HASH_SHA512; 143 data->auth_protocol[0].dhchap.idlist[30] = NVME_AUTH_DHGROUP_NULL; 144 data->auth_protocol[0].dhchap.idlist[31] = NVME_AUTH_DHGROUP_2048; 145 data->auth_protocol[0].dhchap.idlist[32] = NVME_AUTH_DHGROUP_3072; 146 data->auth_protocol[0].dhchap.idlist[33] = NVME_AUTH_DHGROUP_4096; 147 data->auth_protocol[0].dhchap.idlist[34] = NVME_AUTH_DHGROUP_6144; 148 data->auth_protocol[0].dhchap.idlist[35] = NVME_AUTH_DHGROUP_8192; 149 150 return size; 151 } 152 153 static int nvme_auth_process_dhchap_challenge(struct nvme_ctrl *ctrl, 154 struct nvme_dhchap_queue_context *chap) 155 { 156 struct nvmf_auth_dhchap_challenge_data *data = chap->buf; 157 u16 dhvlen = le16_to_cpu(data->dhvlen); 158 size_t size = sizeof(*data) + data->hl + dhvlen; 159 const char *gid_name = nvme_auth_dhgroup_name(data->dhgid); 160 const char *hmac_name, *kpp_name; 161 162 if (size > CHAP_BUF_SIZE) { 163 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 164 return -EINVAL; 165 } 166 167 hmac_name = nvme_auth_hmac_name(data->hashid); 168 if (!hmac_name) { 169 dev_warn(ctrl->device, 170 "qid %d: invalid HASH ID %d\n", 171 chap->qid, data->hashid); 172 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE; 173 return -EPROTO; 174 } 175 176 if (chap->hash_id == data->hashid && chap->shash_tfm && 177 !strcmp(crypto_shash_alg_name(chap->shash_tfm), hmac_name) && 178 crypto_shash_digestsize(chap->shash_tfm) == data->hl) { 179 dev_dbg(ctrl->device, 180 "qid %d: reuse existing hash %s\n", 181 chap->qid, hmac_name); 182 goto select_kpp; 183 } 184 185 /* Reset if hash cannot be reused */ 186 if (chap->shash_tfm) { 187 crypto_free_shash(chap->shash_tfm); 188 chap->hash_id = 0; 189 chap->hash_len = 0; 190 } 191 chap->shash_tfm = crypto_alloc_shash(hmac_name, 0, 192 CRYPTO_ALG_ALLOCATES_MEMORY); 193 if (IS_ERR(chap->shash_tfm)) { 194 dev_warn(ctrl->device, 195 "qid %d: failed to allocate hash %s, error %ld\n", 196 chap->qid, hmac_name, PTR_ERR(chap->shash_tfm)); 197 chap->shash_tfm = NULL; 198 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 199 return -ENOMEM; 200 } 201 202 if (crypto_shash_digestsize(chap->shash_tfm) != data->hl) { 203 dev_warn(ctrl->device, 204 "qid %d: invalid hash length %d\n", 205 chap->qid, data->hl); 206 crypto_free_shash(chap->shash_tfm); 207 chap->shash_tfm = NULL; 208 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE; 209 return -EPROTO; 210 } 211 212 chap->hash_id = data->hashid; 213 chap->hash_len = data->hl; 214 dev_dbg(ctrl->device, "qid %d: selected hash %s\n", 215 chap->qid, hmac_name); 216 217 select_kpp: 218 kpp_name = nvme_auth_dhgroup_kpp(data->dhgid); 219 if (!kpp_name) { 220 dev_warn(ctrl->device, 221 "qid %d: invalid DH group id %d\n", 222 chap->qid, data->dhgid); 223 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE; 224 /* Leave previous dh_tfm intact */ 225 return -EPROTO; 226 } 227 228 if (chap->dhgroup_id == data->dhgid && 229 (data->dhgid == NVME_AUTH_DHGROUP_NULL || chap->dh_tfm)) { 230 dev_dbg(ctrl->device, 231 "qid %d: reuse existing DH group %s\n", 232 chap->qid, gid_name); 233 goto skip_kpp; 234 } 235 236 /* Reset dh_tfm if it can't be reused */ 237 if (chap->dh_tfm) { 238 crypto_free_kpp(chap->dh_tfm); 239 chap->dh_tfm = NULL; 240 } 241 242 if (data->dhgid != NVME_AUTH_DHGROUP_NULL) { 243 if (dhvlen == 0) { 244 dev_warn(ctrl->device, 245 "qid %d: empty DH value\n", 246 chap->qid); 247 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE; 248 return -EPROTO; 249 } 250 251 chap->dh_tfm = crypto_alloc_kpp(kpp_name, 0, 0); 252 if (IS_ERR(chap->dh_tfm)) { 253 int ret = PTR_ERR(chap->dh_tfm); 254 255 dev_warn(ctrl->device, 256 "qid %d: error %d initializing DH group %s\n", 257 chap->qid, ret, gid_name); 258 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE; 259 chap->dh_tfm = NULL; 260 return ret; 261 } 262 dev_dbg(ctrl->device, "qid %d: selected DH group %s\n", 263 chap->qid, gid_name); 264 } else if (dhvlen != 0) { 265 dev_warn(ctrl->device, 266 "qid %d: invalid DH value for NULL DH\n", 267 chap->qid); 268 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 269 return -EPROTO; 270 } 271 chap->dhgroup_id = data->dhgid; 272 273 skip_kpp: 274 chap->s1 = le32_to_cpu(data->seqnum); 275 memcpy(chap->c1, data->cval, chap->hash_len); 276 if (dhvlen) { 277 chap->ctrl_key = kmalloc(dhvlen, GFP_KERNEL); 278 if (!chap->ctrl_key) { 279 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 280 return -ENOMEM; 281 } 282 chap->ctrl_key_len = dhvlen; 283 memcpy(chap->ctrl_key, data->cval + chap->hash_len, 284 dhvlen); 285 dev_dbg(ctrl->device, "ctrl public key %*ph\n", 286 (int)chap->ctrl_key_len, chap->ctrl_key); 287 } 288 289 return 0; 290 } 291 292 static int nvme_auth_set_dhchap_reply_data(struct nvme_ctrl *ctrl, 293 struct nvme_dhchap_queue_context *chap) 294 { 295 struct nvmf_auth_dhchap_reply_data *data = chap->buf; 296 size_t size = sizeof(*data); 297 298 size += 2 * chap->hash_len; 299 300 if (chap->host_key_len) 301 size += chap->host_key_len; 302 303 if (size > CHAP_BUF_SIZE) { 304 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 305 return -EINVAL; 306 } 307 308 memset(chap->buf, 0, size); 309 data->auth_type = NVME_AUTH_DHCHAP_MESSAGES; 310 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_REPLY; 311 data->t_id = cpu_to_le16(chap->transaction); 312 data->hl = chap->hash_len; 313 data->dhvlen = cpu_to_le16(chap->host_key_len); 314 memcpy(data->rval, chap->response, chap->hash_len); 315 if (ctrl->ctrl_key) { 316 chap->bi_directional = true; 317 get_random_bytes(chap->c2, chap->hash_len); 318 data->cvalid = 1; 319 memcpy(data->rval + chap->hash_len, chap->c2, 320 chap->hash_len); 321 dev_dbg(ctrl->device, "%s: qid %d ctrl challenge %*ph\n", 322 __func__, chap->qid, (int)chap->hash_len, chap->c2); 323 } else { 324 memset(chap->c2, 0, chap->hash_len); 325 } 326 chap->s2 = nvme_auth_get_seqnum(); 327 data->seqnum = cpu_to_le32(chap->s2); 328 if (chap->host_key_len) { 329 dev_dbg(ctrl->device, "%s: qid %d host public key %*ph\n", 330 __func__, chap->qid, 331 chap->host_key_len, chap->host_key); 332 memcpy(data->rval + 2 * chap->hash_len, chap->host_key, 333 chap->host_key_len); 334 } 335 336 return size; 337 } 338 339 static int nvme_auth_process_dhchap_success1(struct nvme_ctrl *ctrl, 340 struct nvme_dhchap_queue_context *chap) 341 { 342 struct nvmf_auth_dhchap_success1_data *data = chap->buf; 343 size_t size = sizeof(*data) + chap->hash_len; 344 345 if (size > CHAP_BUF_SIZE) { 346 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 347 return -EINVAL; 348 } 349 350 if (data->hl != chap->hash_len) { 351 dev_warn(ctrl->device, 352 "qid %d: invalid hash length %u\n", 353 chap->qid, data->hl); 354 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE; 355 return -EPROTO; 356 } 357 358 /* Just print out information for the admin queue */ 359 if (chap->qid == 0) 360 dev_info(ctrl->device, 361 "qid 0: authenticated with hash %s dhgroup %s\n", 362 nvme_auth_hmac_name(chap->hash_id), 363 nvme_auth_dhgroup_name(chap->dhgroup_id)); 364 365 if (!data->rvalid) 366 return 0; 367 368 /* Validate controller response */ 369 if (memcmp(chap->response, data->rval, data->hl)) { 370 dev_dbg(ctrl->device, "%s: qid %d ctrl response %*ph\n", 371 __func__, chap->qid, (int)chap->hash_len, data->rval); 372 dev_dbg(ctrl->device, "%s: qid %d host response %*ph\n", 373 __func__, chap->qid, (int)chap->hash_len, 374 chap->response); 375 dev_warn(ctrl->device, 376 "qid %d: controller authentication failed\n", 377 chap->qid); 378 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 379 return -ECONNREFUSED; 380 } 381 382 /* Just print out information for the admin queue */ 383 if (chap->qid == 0) 384 dev_info(ctrl->device, 385 "qid 0: controller authenticated\n"); 386 return 0; 387 } 388 389 static int nvme_auth_set_dhchap_success2_data(struct nvme_ctrl *ctrl, 390 struct nvme_dhchap_queue_context *chap) 391 { 392 struct nvmf_auth_dhchap_success2_data *data = chap->buf; 393 size_t size = sizeof(*data); 394 395 memset(chap->buf, 0, size); 396 data->auth_type = NVME_AUTH_DHCHAP_MESSAGES; 397 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2; 398 data->t_id = cpu_to_le16(chap->transaction); 399 400 return size; 401 } 402 403 static int nvme_auth_set_dhchap_failure2_data(struct nvme_ctrl *ctrl, 404 struct nvme_dhchap_queue_context *chap) 405 { 406 struct nvmf_auth_dhchap_failure_data *data = chap->buf; 407 size_t size = sizeof(*data); 408 409 memset(chap->buf, 0, size); 410 data->auth_type = NVME_AUTH_COMMON_MESSAGES; 411 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_FAILURE2; 412 data->t_id = cpu_to_le16(chap->transaction); 413 data->rescode = NVME_AUTH_DHCHAP_FAILURE_REASON_FAILED; 414 data->rescode_exp = chap->status; 415 416 return size; 417 } 418 419 static int nvme_auth_dhchap_setup_host_response(struct nvme_ctrl *ctrl, 420 struct nvme_dhchap_queue_context *chap) 421 { 422 SHASH_DESC_ON_STACK(shash, chap->shash_tfm); 423 u8 buf[4], *challenge = chap->c1; 424 int ret; 425 426 dev_dbg(ctrl->device, "%s: qid %d host response seq %u transaction %d\n", 427 __func__, chap->qid, chap->s1, chap->transaction); 428 429 if (!chap->transformed_key) { 430 chap->transformed_key = nvme_auth_transform_key(ctrl->host_key, 431 ctrl->opts->host->nqn); 432 if (IS_ERR(chap->transformed_key)) { 433 ret = PTR_ERR(chap->transformed_key); 434 chap->transformed_key = NULL; 435 return ret; 436 } 437 } else { 438 dev_dbg(ctrl->device, "%s: qid %d re-using host response\n", 439 __func__, chap->qid); 440 } 441 442 ret = crypto_shash_setkey(chap->shash_tfm, 443 chap->transformed_key->key, chap->transformed_key->len); 444 if (ret) { 445 dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n", 446 chap->qid, ret); 447 goto out; 448 } 449 450 if (chap->dh_tfm) { 451 challenge = kmalloc(chap->hash_len, GFP_KERNEL); 452 if (!challenge) { 453 ret = -ENOMEM; 454 goto out; 455 } 456 ret = nvme_auth_augmented_challenge(chap->hash_id, 457 chap->sess_key, 458 chap->sess_key_len, 459 chap->c1, challenge, 460 chap->hash_len); 461 if (ret) 462 goto out; 463 } 464 465 shash->tfm = chap->shash_tfm; 466 ret = crypto_shash_init(shash); 467 if (ret) 468 goto out; 469 ret = crypto_shash_update(shash, challenge, chap->hash_len); 470 if (ret) 471 goto out; 472 put_unaligned_le32(chap->s1, buf); 473 ret = crypto_shash_update(shash, buf, 4); 474 if (ret) 475 goto out; 476 put_unaligned_le16(chap->transaction, buf); 477 ret = crypto_shash_update(shash, buf, 2); 478 if (ret) 479 goto out; 480 memset(buf, 0, sizeof(buf)); 481 ret = crypto_shash_update(shash, buf, 1); 482 if (ret) 483 goto out; 484 ret = crypto_shash_update(shash, "HostHost", 8); 485 if (ret) 486 goto out; 487 ret = crypto_shash_update(shash, ctrl->opts->host->nqn, 488 strlen(ctrl->opts->host->nqn)); 489 if (ret) 490 goto out; 491 ret = crypto_shash_update(shash, buf, 1); 492 if (ret) 493 goto out; 494 ret = crypto_shash_update(shash, ctrl->opts->subsysnqn, 495 strlen(ctrl->opts->subsysnqn)); 496 if (ret) 497 goto out; 498 ret = crypto_shash_final(shash, chap->response); 499 out: 500 if (challenge != chap->c1) 501 kfree(challenge); 502 return ret; 503 } 504 505 static int nvme_auth_dhchap_setup_ctrl_response(struct nvme_ctrl *ctrl, 506 struct nvme_dhchap_queue_context *chap) 507 { 508 SHASH_DESC_ON_STACK(shash, chap->shash_tfm); 509 struct nvme_dhchap_key *transformed_key; 510 u8 buf[4], *challenge = chap->c2; 511 int ret; 512 513 transformed_key = nvme_auth_transform_key(ctrl->ctrl_key, 514 ctrl->opts->subsysnqn); 515 if (IS_ERR(transformed_key)) { 516 ret = PTR_ERR(transformed_key); 517 return ret; 518 } 519 520 ret = crypto_shash_setkey(chap->shash_tfm, 521 transformed_key->key, transformed_key->len); 522 if (ret) { 523 dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n", 524 chap->qid, ret); 525 goto out; 526 } 527 528 if (chap->dh_tfm) { 529 challenge = kmalloc(chap->hash_len, GFP_KERNEL); 530 if (!challenge) { 531 ret = -ENOMEM; 532 goto out; 533 } 534 ret = nvme_auth_augmented_challenge(chap->hash_id, 535 chap->sess_key, 536 chap->sess_key_len, 537 chap->c2, challenge, 538 chap->hash_len); 539 if (ret) 540 goto out; 541 } 542 dev_dbg(ctrl->device, "%s: qid %d ctrl response seq %u transaction %d\n", 543 __func__, chap->qid, chap->s2, chap->transaction); 544 dev_dbg(ctrl->device, "%s: qid %d challenge %*ph\n", 545 __func__, chap->qid, (int)chap->hash_len, challenge); 546 dev_dbg(ctrl->device, "%s: qid %d subsysnqn %s\n", 547 __func__, chap->qid, ctrl->opts->subsysnqn); 548 dev_dbg(ctrl->device, "%s: qid %d hostnqn %s\n", 549 __func__, chap->qid, ctrl->opts->host->nqn); 550 shash->tfm = chap->shash_tfm; 551 ret = crypto_shash_init(shash); 552 if (ret) 553 goto out; 554 ret = crypto_shash_update(shash, challenge, chap->hash_len); 555 if (ret) 556 goto out; 557 put_unaligned_le32(chap->s2, buf); 558 ret = crypto_shash_update(shash, buf, 4); 559 if (ret) 560 goto out; 561 put_unaligned_le16(chap->transaction, buf); 562 ret = crypto_shash_update(shash, buf, 2); 563 if (ret) 564 goto out; 565 memset(buf, 0, 4); 566 ret = crypto_shash_update(shash, buf, 1); 567 if (ret) 568 goto out; 569 ret = crypto_shash_update(shash, "Controller", 10); 570 if (ret) 571 goto out; 572 ret = crypto_shash_update(shash, ctrl->opts->subsysnqn, 573 strlen(ctrl->opts->subsysnqn)); 574 if (ret) 575 goto out; 576 ret = crypto_shash_update(shash, buf, 1); 577 if (ret) 578 goto out; 579 ret = crypto_shash_update(shash, ctrl->opts->host->nqn, 580 strlen(ctrl->opts->host->nqn)); 581 if (ret) 582 goto out; 583 ret = crypto_shash_final(shash, chap->response); 584 out: 585 if (challenge != chap->c2) 586 kfree(challenge); 587 nvme_auth_free_key(transformed_key); 588 return ret; 589 } 590 591 static int nvme_auth_dhchap_exponential(struct nvme_ctrl *ctrl, 592 struct nvme_dhchap_queue_context *chap) 593 { 594 int ret; 595 596 if (chap->host_key && chap->host_key_len) { 597 dev_dbg(ctrl->device, 598 "qid %d: reusing host key\n", chap->qid); 599 goto gen_sesskey; 600 } 601 ret = nvme_auth_gen_privkey(chap->dh_tfm, chap->dhgroup_id); 602 if (ret < 0) { 603 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 604 return ret; 605 } 606 607 chap->host_key_len = crypto_kpp_maxsize(chap->dh_tfm); 608 609 chap->host_key = kzalloc(chap->host_key_len, GFP_KERNEL); 610 if (!chap->host_key) { 611 chap->host_key_len = 0; 612 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 613 return -ENOMEM; 614 } 615 ret = nvme_auth_gen_pubkey(chap->dh_tfm, 616 chap->host_key, chap->host_key_len); 617 if (ret) { 618 dev_dbg(ctrl->device, 619 "failed to generate public key, error %d\n", ret); 620 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 621 return ret; 622 } 623 624 gen_sesskey: 625 chap->sess_key_len = chap->host_key_len; 626 chap->sess_key = kmalloc(chap->sess_key_len, GFP_KERNEL); 627 if (!chap->sess_key) { 628 chap->sess_key_len = 0; 629 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 630 return -ENOMEM; 631 } 632 633 ret = nvme_auth_gen_shared_secret(chap->dh_tfm, 634 chap->ctrl_key, chap->ctrl_key_len, 635 chap->sess_key, chap->sess_key_len); 636 if (ret) { 637 dev_dbg(ctrl->device, 638 "failed to generate shared secret, error %d\n", ret); 639 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 640 return ret; 641 } 642 dev_dbg(ctrl->device, "shared secret %*ph\n", 643 (int)chap->sess_key_len, chap->sess_key); 644 return 0; 645 } 646 647 static void nvme_auth_reset_dhchap(struct nvme_dhchap_queue_context *chap) 648 { 649 nvme_auth_free_key(chap->transformed_key); 650 chap->transformed_key = NULL; 651 kfree_sensitive(chap->host_key); 652 chap->host_key = NULL; 653 chap->host_key_len = 0; 654 kfree_sensitive(chap->ctrl_key); 655 chap->ctrl_key = NULL; 656 chap->ctrl_key_len = 0; 657 kfree_sensitive(chap->sess_key); 658 chap->sess_key = NULL; 659 chap->sess_key_len = 0; 660 chap->status = 0; 661 chap->error = 0; 662 chap->s1 = 0; 663 chap->s2 = 0; 664 chap->bi_directional = false; 665 chap->transaction = 0; 666 memset(chap->c1, 0, sizeof(chap->c1)); 667 memset(chap->c2, 0, sizeof(chap->c2)); 668 mempool_free(chap->buf, nvme_chap_buf_pool); 669 chap->buf = NULL; 670 } 671 672 static void nvme_auth_free_dhchap(struct nvme_dhchap_queue_context *chap) 673 { 674 nvme_auth_reset_dhchap(chap); 675 if (chap->shash_tfm) 676 crypto_free_shash(chap->shash_tfm); 677 if (chap->dh_tfm) 678 crypto_free_kpp(chap->dh_tfm); 679 } 680 681 static void nvme_queue_auth_work(struct work_struct *work) 682 { 683 struct nvme_dhchap_queue_context *chap = 684 container_of(work, struct nvme_dhchap_queue_context, auth_work); 685 struct nvme_ctrl *ctrl = chap->ctrl; 686 size_t tl; 687 int ret = 0; 688 689 /* 690 * Allocate a large enough buffer for the entire negotiation: 691 * 4k is enough to ffdhe8192. 692 */ 693 chap->buf = mempool_alloc(nvme_chap_buf_pool, GFP_KERNEL); 694 if (!chap->buf) { 695 chap->error = -ENOMEM; 696 return; 697 } 698 699 chap->transaction = ctrl->transaction++; 700 701 /* DH-HMAC-CHAP Step 1: send negotiate */ 702 dev_dbg(ctrl->device, "%s: qid %d send negotiate\n", 703 __func__, chap->qid); 704 ret = nvme_auth_set_dhchap_negotiate_data(ctrl, chap); 705 if (ret < 0) { 706 chap->error = ret; 707 return; 708 } 709 tl = ret; 710 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true); 711 if (ret) { 712 chap->error = ret; 713 return; 714 } 715 716 /* DH-HMAC-CHAP Step 2: receive challenge */ 717 dev_dbg(ctrl->device, "%s: qid %d receive challenge\n", 718 __func__, chap->qid); 719 720 memset(chap->buf, 0, CHAP_BUF_SIZE); 721 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, CHAP_BUF_SIZE, 722 false); 723 if (ret) { 724 dev_warn(ctrl->device, 725 "qid %d failed to receive challenge, %s %d\n", 726 chap->qid, ret < 0 ? "error" : "nvme status", ret); 727 chap->error = ret; 728 return; 729 } 730 ret = nvme_auth_receive_validate(ctrl, chap->qid, chap->buf, chap->transaction, 731 NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE); 732 if (ret) { 733 chap->status = ret; 734 chap->error = -ECONNREFUSED; 735 return; 736 } 737 738 ret = nvme_auth_process_dhchap_challenge(ctrl, chap); 739 if (ret) { 740 /* Invalid challenge parameters */ 741 chap->error = ret; 742 goto fail2; 743 } 744 745 if (chap->ctrl_key_len) { 746 dev_dbg(ctrl->device, 747 "%s: qid %d DH exponential\n", 748 __func__, chap->qid); 749 ret = nvme_auth_dhchap_exponential(ctrl, chap); 750 if (ret) { 751 chap->error = ret; 752 goto fail2; 753 } 754 } 755 756 dev_dbg(ctrl->device, "%s: qid %d host response\n", 757 __func__, chap->qid); 758 mutex_lock(&ctrl->dhchap_auth_mutex); 759 ret = nvme_auth_dhchap_setup_host_response(ctrl, chap); 760 if (ret) { 761 mutex_unlock(&ctrl->dhchap_auth_mutex); 762 chap->error = ret; 763 goto fail2; 764 } 765 mutex_unlock(&ctrl->dhchap_auth_mutex); 766 767 /* DH-HMAC-CHAP Step 3: send reply */ 768 dev_dbg(ctrl->device, "%s: qid %d send reply\n", 769 __func__, chap->qid); 770 ret = nvme_auth_set_dhchap_reply_data(ctrl, chap); 771 if (ret < 0) { 772 chap->error = ret; 773 goto fail2; 774 } 775 776 tl = ret; 777 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true); 778 if (ret) { 779 chap->error = ret; 780 goto fail2; 781 } 782 783 /* DH-HMAC-CHAP Step 4: receive success1 */ 784 dev_dbg(ctrl->device, "%s: qid %d receive success1\n", 785 __func__, chap->qid); 786 787 memset(chap->buf, 0, CHAP_BUF_SIZE); 788 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, CHAP_BUF_SIZE, 789 false); 790 if (ret) { 791 dev_warn(ctrl->device, 792 "qid %d failed to receive success1, %s %d\n", 793 chap->qid, ret < 0 ? "error" : "nvme status", ret); 794 chap->error = ret; 795 return; 796 } 797 ret = nvme_auth_receive_validate(ctrl, chap->qid, 798 chap->buf, chap->transaction, 799 NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1); 800 if (ret) { 801 chap->status = ret; 802 chap->error = -ECONNREFUSED; 803 return; 804 } 805 806 mutex_lock(&ctrl->dhchap_auth_mutex); 807 if (ctrl->ctrl_key) { 808 dev_dbg(ctrl->device, 809 "%s: qid %d controller response\n", 810 __func__, chap->qid); 811 ret = nvme_auth_dhchap_setup_ctrl_response(ctrl, chap); 812 if (ret) { 813 mutex_unlock(&ctrl->dhchap_auth_mutex); 814 chap->error = ret; 815 goto fail2; 816 } 817 } 818 mutex_unlock(&ctrl->dhchap_auth_mutex); 819 820 ret = nvme_auth_process_dhchap_success1(ctrl, chap); 821 if (ret) { 822 /* Controller authentication failed */ 823 chap->error = -ECONNREFUSED; 824 goto fail2; 825 } 826 827 if (chap->bi_directional) { 828 /* DH-HMAC-CHAP Step 5: send success2 */ 829 dev_dbg(ctrl->device, "%s: qid %d send success2\n", 830 __func__, chap->qid); 831 tl = nvme_auth_set_dhchap_success2_data(ctrl, chap); 832 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true); 833 if (ret) 834 chap->error = ret; 835 } 836 if (!ret) { 837 chap->error = 0; 838 return; 839 } 840 841 fail2: 842 dev_dbg(ctrl->device, "%s: qid %d send failure2, status %x\n", 843 __func__, chap->qid, chap->status); 844 tl = nvme_auth_set_dhchap_failure2_data(ctrl, chap); 845 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true); 846 /* 847 * only update error if send failure2 failed and no other 848 * error had been set during authentication. 849 */ 850 if (ret && !chap->error) 851 chap->error = ret; 852 } 853 854 int nvme_auth_negotiate(struct nvme_ctrl *ctrl, int qid) 855 { 856 struct nvme_dhchap_queue_context *chap; 857 858 if (!ctrl->host_key) { 859 dev_warn(ctrl->device, "qid %d: no key\n", qid); 860 return -ENOKEY; 861 } 862 863 if (ctrl->opts->dhchap_ctrl_secret && !ctrl->ctrl_key) { 864 dev_warn(ctrl->device, "qid %d: invalid ctrl key\n", qid); 865 return -ENOKEY; 866 } 867 868 chap = &ctrl->dhchap_ctxs[qid]; 869 cancel_work_sync(&chap->auth_work); 870 queue_work(nvme_auth_wq, &chap->auth_work); 871 return 0; 872 } 873 EXPORT_SYMBOL_GPL(nvme_auth_negotiate); 874 875 int nvme_auth_wait(struct nvme_ctrl *ctrl, int qid) 876 { 877 struct nvme_dhchap_queue_context *chap; 878 int ret; 879 880 chap = &ctrl->dhchap_ctxs[qid]; 881 flush_work(&chap->auth_work); 882 ret = chap->error; 883 /* clear sensitive info */ 884 nvme_auth_reset_dhchap(chap); 885 return ret; 886 } 887 EXPORT_SYMBOL_GPL(nvme_auth_wait); 888 889 static void nvme_ctrl_auth_work(struct work_struct *work) 890 { 891 struct nvme_ctrl *ctrl = 892 container_of(work, struct nvme_ctrl, dhchap_auth_work); 893 int ret, q; 894 895 /* 896 * If the ctrl is no connected, bail as reconnect will handle 897 * authentication. 898 */ 899 if (ctrl->state != NVME_CTRL_LIVE) 900 return; 901 902 /* Authenticate admin queue first */ 903 ret = nvme_auth_negotiate(ctrl, 0); 904 if (ret) { 905 dev_warn(ctrl->device, 906 "qid 0: error %d setting up authentication\n", ret); 907 return; 908 } 909 ret = nvme_auth_wait(ctrl, 0); 910 if (ret) { 911 dev_warn(ctrl->device, 912 "qid 0: authentication failed\n"); 913 return; 914 } 915 916 for (q = 1; q < ctrl->queue_count; q++) { 917 ret = nvme_auth_negotiate(ctrl, q); 918 if (ret) { 919 dev_warn(ctrl->device, 920 "qid %d: error %d setting up authentication\n", 921 q, ret); 922 break; 923 } 924 } 925 926 /* 927 * Failure is a soft-state; credentials remain valid until 928 * the controller terminates the connection. 929 */ 930 for (q = 1; q < ctrl->queue_count; q++) { 931 ret = nvme_auth_wait(ctrl, q); 932 if (ret) 933 dev_warn(ctrl->device, 934 "qid %d: authentication failed\n", q); 935 } 936 } 937 938 int nvme_auth_init_ctrl(struct nvme_ctrl *ctrl) 939 { 940 struct nvme_dhchap_queue_context *chap; 941 int i, ret; 942 943 mutex_init(&ctrl->dhchap_auth_mutex); 944 INIT_WORK(&ctrl->dhchap_auth_work, nvme_ctrl_auth_work); 945 if (!ctrl->opts) 946 return 0; 947 ret = nvme_auth_generate_key(ctrl->opts->dhchap_secret, 948 &ctrl->host_key); 949 if (ret) 950 return ret; 951 ret = nvme_auth_generate_key(ctrl->opts->dhchap_ctrl_secret, 952 &ctrl->ctrl_key); 953 if (ret) 954 goto err_free_dhchap_secret; 955 956 if (!ctrl->opts->dhchap_secret && !ctrl->opts->dhchap_ctrl_secret) 957 return 0; 958 959 ctrl->dhchap_ctxs = kvcalloc(ctrl_max_dhchaps(ctrl), 960 sizeof(*chap), GFP_KERNEL); 961 if (!ctrl->dhchap_ctxs) { 962 ret = -ENOMEM; 963 goto err_free_dhchap_ctrl_secret; 964 } 965 966 for (i = 0; i < ctrl_max_dhchaps(ctrl); i++) { 967 chap = &ctrl->dhchap_ctxs[i]; 968 chap->qid = i; 969 chap->ctrl = ctrl; 970 INIT_WORK(&chap->auth_work, nvme_queue_auth_work); 971 } 972 973 return 0; 974 err_free_dhchap_ctrl_secret: 975 nvme_auth_free_key(ctrl->ctrl_key); 976 ctrl->ctrl_key = NULL; 977 err_free_dhchap_secret: 978 nvme_auth_free_key(ctrl->host_key); 979 ctrl->host_key = NULL; 980 return ret; 981 } 982 EXPORT_SYMBOL_GPL(nvme_auth_init_ctrl); 983 984 void nvme_auth_stop(struct nvme_ctrl *ctrl) 985 { 986 cancel_work_sync(&ctrl->dhchap_auth_work); 987 } 988 EXPORT_SYMBOL_GPL(nvme_auth_stop); 989 990 void nvme_auth_free(struct nvme_ctrl *ctrl) 991 { 992 int i; 993 994 if (ctrl->dhchap_ctxs) { 995 for (i = 0; i < ctrl_max_dhchaps(ctrl); i++) 996 nvme_auth_free_dhchap(&ctrl->dhchap_ctxs[i]); 997 kfree(ctrl->dhchap_ctxs); 998 } 999 if (ctrl->host_key) { 1000 nvme_auth_free_key(ctrl->host_key); 1001 ctrl->host_key = NULL; 1002 } 1003 if (ctrl->ctrl_key) { 1004 nvme_auth_free_key(ctrl->ctrl_key); 1005 ctrl->ctrl_key = NULL; 1006 } 1007 } 1008 EXPORT_SYMBOL_GPL(nvme_auth_free); 1009 1010 int __init nvme_init_auth(void) 1011 { 1012 nvme_auth_wq = alloc_workqueue("nvme-auth-wq", 1013 WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0); 1014 if (!nvme_auth_wq) 1015 return -ENOMEM; 1016 1017 nvme_chap_buf_cache = kmem_cache_create("nvme-chap-buf-cache", 1018 CHAP_BUF_SIZE, 0, SLAB_HWCACHE_ALIGN, NULL); 1019 if (!nvme_chap_buf_cache) 1020 goto err_destroy_workqueue; 1021 1022 nvme_chap_buf_pool = mempool_create(16, mempool_alloc_slab, 1023 mempool_free_slab, nvme_chap_buf_cache); 1024 if (!nvme_chap_buf_pool) 1025 goto err_destroy_chap_buf_cache; 1026 1027 return 0; 1028 err_destroy_chap_buf_cache: 1029 kmem_cache_destroy(nvme_chap_buf_cache); 1030 err_destroy_workqueue: 1031 destroy_workqueue(nvme_auth_wq); 1032 return -ENOMEM; 1033 } 1034 1035 void __exit nvme_exit_auth(void) 1036 { 1037 mempool_destroy(nvme_chap_buf_pool); 1038 kmem_cache_destroy(nvme_chap_buf_cache); 1039 destroy_workqueue(nvme_auth_wq); 1040 } 1041