1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2020 Hannes Reinecke, SUSE Linux 4 */ 5 6 #include <linux/crc32.h> 7 #include <linux/base64.h> 8 #include <linux/prandom.h> 9 #include <linux/unaligned.h> 10 #include <crypto/dh.h> 11 #include <crypto/utils.h> 12 #include "nvme.h" 13 #include "fabrics.h" 14 #include <linux/nvme-auth.h> 15 #include <linux/nvme-keyring.h> 16 17 #define CHAP_BUF_SIZE 4096 18 static struct kmem_cache *nvme_chap_buf_cache; 19 static mempool_t *nvme_chap_buf_pool; 20 21 struct nvme_dhchap_queue_context { 22 struct list_head entry; 23 struct work_struct auth_work; 24 struct nvme_ctrl *ctrl; 25 struct crypto_kpp *dh_tfm; 26 struct nvme_dhchap_key *transformed_key; 27 void *buf; 28 int qid; 29 int error; 30 u32 s1; 31 u32 s2; 32 bool bi_directional; 33 bool authenticated; 34 u16 transaction; 35 u8 status; 36 u8 dhgroup_id; 37 u8 hash_id; 38 u8 sc_c; 39 size_t hash_len; 40 u8 c1[NVME_AUTH_MAX_DIGEST_SIZE]; 41 u8 c2[NVME_AUTH_MAX_DIGEST_SIZE]; 42 u8 response[NVME_AUTH_MAX_DIGEST_SIZE]; 43 u8 *ctrl_key; 44 u8 *host_key; 45 u8 *sess_key; 46 int ctrl_key_len; 47 int host_key_len; 48 int sess_key_len; 49 }; 50 51 static struct workqueue_struct *nvme_auth_wq; 52 53 static inline int ctrl_max_dhchaps(struct nvme_ctrl *ctrl) 54 { 55 return ctrl->opts->nr_io_queues + ctrl->opts->nr_write_queues + 56 ctrl->opts->nr_poll_queues + 1; 57 } 58 59 static int nvme_auth_submit(struct nvme_ctrl *ctrl, int qid, 60 void *data, size_t data_len, bool auth_send) 61 { 62 struct nvme_command cmd = {}; 63 nvme_submit_flags_t flags = NVME_SUBMIT_RETRY; 64 struct request_queue *q = ctrl->fabrics_q; 65 int ret; 66 67 if (qid != 0) { 68 flags |= NVME_SUBMIT_NOWAIT | NVME_SUBMIT_RESERVED; 69 q = ctrl->connect_q; 70 } 71 72 cmd.auth_common.opcode = nvme_fabrics_command; 73 cmd.auth_common.secp = NVME_AUTH_DHCHAP_PROTOCOL_IDENTIFIER; 74 cmd.auth_common.spsp0 = 0x01; 75 cmd.auth_common.spsp1 = 0x01; 76 if (auth_send) { 77 cmd.auth_send.fctype = nvme_fabrics_type_auth_send; 78 cmd.auth_send.tl = cpu_to_le32(data_len); 79 } else { 80 cmd.auth_receive.fctype = nvme_fabrics_type_auth_receive; 81 cmd.auth_receive.al = cpu_to_le32(data_len); 82 } 83 84 ret = __nvme_submit_sync_cmd(q, &cmd, NULL, data, data_len, 85 qid == 0 ? NVME_QID_ANY : qid, flags); 86 if (ret > 0) 87 dev_warn(ctrl->device, 88 "qid %d auth_send failed with status %d\n", qid, ret); 89 else if (ret < 0) 90 dev_err(ctrl->device, 91 "qid %d auth_send failed with error %d\n", qid, ret); 92 return ret; 93 } 94 95 static int nvme_auth_receive_validate(struct nvme_ctrl *ctrl, int qid, 96 struct nvmf_auth_dhchap_failure_data *data, 97 u16 transaction, u8 expected_msg) 98 { 99 dev_dbg(ctrl->device, "%s: qid %d auth_type %d auth_id %x\n", 100 __func__, qid, data->auth_type, data->auth_id); 101 102 if (data->auth_type == NVME_AUTH_COMMON_MESSAGES && 103 data->auth_id == NVME_AUTH_DHCHAP_MESSAGE_FAILURE1) { 104 return data->rescode_exp; 105 } 106 if (data->auth_type != NVME_AUTH_DHCHAP_MESSAGES || 107 data->auth_id != expected_msg) { 108 dev_warn(ctrl->device, 109 "qid %d invalid message %02x/%02x\n", 110 qid, data->auth_type, data->auth_id); 111 return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE; 112 } 113 if (le16_to_cpu(data->t_id) != transaction) { 114 dev_warn(ctrl->device, 115 "qid %d invalid transaction ID %d\n", 116 qid, le16_to_cpu(data->t_id)); 117 return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE; 118 } 119 return 0; 120 } 121 122 static int nvme_auth_set_dhchap_negotiate_data(struct nvme_ctrl *ctrl, 123 struct nvme_dhchap_queue_context *chap) 124 { 125 struct nvmf_auth_dhchap_negotiate_data *data = chap->buf; 126 size_t size = sizeof(*data) + sizeof(union nvmf_auth_protocol); 127 u8 dh_list_offset = NVME_AUTH_DHCHAP_MAX_DH_IDS; 128 u8 *idlist = data->auth_protocol[0].dhchap.idlist; 129 130 if (size > CHAP_BUF_SIZE) { 131 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 132 return -EINVAL; 133 } 134 memset((u8 *)chap->buf, 0, size); 135 data->auth_type = NVME_AUTH_COMMON_MESSAGES; 136 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE; 137 data->t_id = cpu_to_le16(chap->transaction); 138 if (ctrl->opts->concat && chap->qid == 0) { 139 if (ctrl->opts->tls_key) 140 data->sc_c = NVME_AUTH_SECP_REPLACETLSPSK; 141 else 142 data->sc_c = NVME_AUTH_SECP_NEWTLSPSK; 143 } else 144 data->sc_c = NVME_AUTH_SECP_NOSC; 145 chap->sc_c = data->sc_c; 146 data->napd = 1; 147 data->auth_protocol[0].dhchap.authid = NVME_AUTH_DHCHAP_AUTH_ID; 148 data->auth_protocol[0].dhchap.halen = 3; 149 idlist[0] = NVME_AUTH_HASH_SHA256; 150 idlist[1] = NVME_AUTH_HASH_SHA384; 151 idlist[2] = NVME_AUTH_HASH_SHA512; 152 if (chap->sc_c == NVME_AUTH_SECP_NOSC) 153 idlist[dh_list_offset++] = NVME_AUTH_DHGROUP_NULL; 154 idlist[dh_list_offset++] = NVME_AUTH_DHGROUP_2048; 155 idlist[dh_list_offset++] = NVME_AUTH_DHGROUP_3072; 156 idlist[dh_list_offset++] = NVME_AUTH_DHGROUP_4096; 157 idlist[dh_list_offset++] = NVME_AUTH_DHGROUP_6144; 158 idlist[dh_list_offset++] = NVME_AUTH_DHGROUP_8192; 159 data->auth_protocol[0].dhchap.dhlen = 160 dh_list_offset - NVME_AUTH_DHCHAP_MAX_DH_IDS; 161 162 return size; 163 } 164 165 static int nvme_auth_process_dhchap_challenge(struct nvme_ctrl *ctrl, 166 struct nvme_dhchap_queue_context *chap) 167 { 168 struct nvmf_auth_dhchap_challenge_data *data = chap->buf; 169 u16 dhvlen = le16_to_cpu(data->dhvlen); 170 size_t size = sizeof(*data) + data->hl + dhvlen; 171 const char *gid_name = nvme_auth_dhgroup_name(data->dhgid); 172 const char *hmac_name, *kpp_name; 173 174 if (size > CHAP_BUF_SIZE) { 175 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 176 return -EINVAL; 177 } 178 179 hmac_name = nvme_auth_hmac_name(data->hashid); 180 if (!hmac_name) { 181 dev_warn(ctrl->device, 182 "qid %d: invalid HASH ID %d\n", 183 chap->qid, data->hashid); 184 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE; 185 return -EPROTO; 186 } 187 188 if (chap->hash_id == data->hashid && chap->hash_len == data->hl) { 189 dev_dbg(ctrl->device, 190 "qid %d: reuse existing hash %s\n", 191 chap->qid, hmac_name); 192 goto select_kpp; 193 } 194 195 if (nvme_auth_hmac_hash_len(data->hashid) != data->hl) { 196 dev_warn(ctrl->device, 197 "qid %d: invalid hash length %d\n", 198 chap->qid, data->hl); 199 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE; 200 return -EPROTO; 201 } 202 203 chap->hash_id = data->hashid; 204 chap->hash_len = data->hl; 205 dev_dbg(ctrl->device, "qid %d: selected hash %s\n", 206 chap->qid, hmac_name); 207 208 select_kpp: 209 kpp_name = nvme_auth_dhgroup_kpp(data->dhgid); 210 if (!kpp_name) { 211 dev_warn(ctrl->device, 212 "qid %d: invalid DH group id %d\n", 213 chap->qid, data->dhgid); 214 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE; 215 /* Leave previous dh_tfm intact */ 216 return -EPROTO; 217 } 218 219 if (chap->dhgroup_id == data->dhgid && 220 (data->dhgid == NVME_AUTH_DHGROUP_NULL || chap->dh_tfm)) { 221 dev_dbg(ctrl->device, 222 "qid %d: reuse existing DH group %s\n", 223 chap->qid, gid_name); 224 goto skip_kpp; 225 } 226 227 /* Reset dh_tfm if it can't be reused */ 228 if (chap->dh_tfm) { 229 crypto_free_kpp(chap->dh_tfm); 230 chap->dh_tfm = NULL; 231 } 232 233 if (data->dhgid != NVME_AUTH_DHGROUP_NULL) { 234 if (dhvlen == 0) { 235 dev_warn(ctrl->device, 236 "qid %d: empty DH value\n", 237 chap->qid); 238 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE; 239 return -EPROTO; 240 } 241 242 chap->dh_tfm = crypto_alloc_kpp(kpp_name, 0, 0); 243 if (IS_ERR(chap->dh_tfm)) { 244 int ret = PTR_ERR(chap->dh_tfm); 245 246 dev_warn(ctrl->device, 247 "qid %d: error %d initializing DH group %s\n", 248 chap->qid, ret, gid_name); 249 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE; 250 chap->dh_tfm = NULL; 251 return ret; 252 } 253 dev_dbg(ctrl->device, "qid %d: selected DH group %s\n", 254 chap->qid, gid_name); 255 } else if (dhvlen != 0) { 256 dev_warn(ctrl->device, 257 "qid %d: invalid DH value for NULL DH\n", 258 chap->qid); 259 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 260 return -EPROTO; 261 } 262 chap->dhgroup_id = data->dhgid; 263 264 skip_kpp: 265 chap->s1 = le32_to_cpu(data->seqnum); 266 memcpy(chap->c1, data->cval, chap->hash_len); 267 if (dhvlen) { 268 chap->ctrl_key = kmalloc(dhvlen, GFP_KERNEL); 269 if (!chap->ctrl_key) { 270 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 271 return -ENOMEM; 272 } 273 chap->ctrl_key_len = dhvlen; 274 memcpy(chap->ctrl_key, data->cval + chap->hash_len, 275 dhvlen); 276 dev_dbg(ctrl->device, "ctrl public key %*ph\n", 277 (int)chap->ctrl_key_len, chap->ctrl_key); 278 } 279 280 return 0; 281 } 282 283 static int nvme_auth_set_dhchap_reply_data(struct nvme_ctrl *ctrl, 284 struct nvme_dhchap_queue_context *chap) 285 { 286 struct nvmf_auth_dhchap_reply_data *data = chap->buf; 287 size_t size = sizeof(*data); 288 289 size += 2 * chap->hash_len; 290 291 if (chap->host_key_len) 292 size += chap->host_key_len; 293 294 if (size > CHAP_BUF_SIZE) { 295 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 296 return -EINVAL; 297 } 298 299 memset(chap->buf, 0, size); 300 data->auth_type = NVME_AUTH_DHCHAP_MESSAGES; 301 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_REPLY; 302 data->t_id = cpu_to_le16(chap->transaction); 303 data->hl = chap->hash_len; 304 data->dhvlen = cpu_to_le16(chap->host_key_len); 305 memcpy(data->rval, chap->response, chap->hash_len); 306 if (ctrl->ctrl_key) 307 chap->bi_directional = true; 308 if (ctrl->ctrl_key || ctrl->opts->concat) { 309 get_random_bytes(chap->c2, chap->hash_len); 310 data->cvalid = 1; 311 memcpy(data->rval + chap->hash_len, chap->c2, 312 chap->hash_len); 313 dev_dbg(ctrl->device, "%s: qid %d ctrl challenge %*ph\n", 314 __func__, chap->qid, (int)chap->hash_len, chap->c2); 315 } else { 316 memset(chap->c2, 0, chap->hash_len); 317 } 318 if (ctrl->opts->concat) { 319 chap->s2 = 0; 320 chap->bi_directional = false; 321 } else 322 chap->s2 = nvme_auth_get_seqnum(); 323 data->seqnum = cpu_to_le32(chap->s2); 324 if (chap->host_key_len) { 325 dev_dbg(ctrl->device, "%s: qid %d host public key %*ph\n", 326 __func__, chap->qid, 327 chap->host_key_len, chap->host_key); 328 memcpy(data->rval + 2 * chap->hash_len, chap->host_key, 329 chap->host_key_len); 330 } 331 332 return size; 333 } 334 335 static int nvme_auth_process_dhchap_success1(struct nvme_ctrl *ctrl, 336 struct nvme_dhchap_queue_context *chap) 337 { 338 struct nvmf_auth_dhchap_success1_data *data = chap->buf; 339 size_t size = sizeof(*data) + chap->hash_len; 340 341 if (size > CHAP_BUF_SIZE) { 342 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 343 return -EINVAL; 344 } 345 346 if (data->hl != chap->hash_len) { 347 dev_warn(ctrl->device, 348 "qid %d: invalid hash length %u\n", 349 chap->qid, data->hl); 350 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE; 351 return -EPROTO; 352 } 353 354 /* Just print out information for the admin queue */ 355 if (chap->qid == 0) 356 dev_info(ctrl->device, 357 "qid 0: authenticated with hash %s dhgroup %s\n", 358 nvme_auth_hmac_name(chap->hash_id), 359 nvme_auth_dhgroup_name(chap->dhgroup_id)); 360 361 if (!data->rvalid) 362 return 0; 363 364 /* Validate controller response */ 365 if (crypto_memneq(chap->response, data->rval, data->hl)) { 366 dev_dbg(ctrl->device, "%s: qid %d ctrl response %*ph\n", 367 __func__, chap->qid, (int)chap->hash_len, data->rval); 368 dev_dbg(ctrl->device, "%s: qid %d host response %*ph\n", 369 __func__, chap->qid, (int)chap->hash_len, 370 chap->response); 371 dev_warn(ctrl->device, 372 "qid %d: controller authentication failed\n", 373 chap->qid); 374 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 375 return -ECONNREFUSED; 376 } 377 378 /* Just print out information for the admin queue */ 379 if (chap->qid == 0) 380 dev_info(ctrl->device, 381 "qid 0: controller authenticated\n"); 382 return 0; 383 } 384 385 static int nvme_auth_set_dhchap_success2_data(struct nvme_ctrl *ctrl, 386 struct nvme_dhchap_queue_context *chap) 387 { 388 struct nvmf_auth_dhchap_success2_data *data = chap->buf; 389 size_t size = sizeof(*data); 390 391 memset(chap->buf, 0, size); 392 data->auth_type = NVME_AUTH_DHCHAP_MESSAGES; 393 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2; 394 data->t_id = cpu_to_le16(chap->transaction); 395 396 return size; 397 } 398 399 static int nvme_auth_set_dhchap_failure2_data(struct nvme_ctrl *ctrl, 400 struct nvme_dhchap_queue_context *chap) 401 { 402 struct nvmf_auth_dhchap_failure_data *data = chap->buf; 403 size_t size = sizeof(*data); 404 405 memset(chap->buf, 0, size); 406 data->auth_type = NVME_AUTH_COMMON_MESSAGES; 407 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_FAILURE2; 408 data->t_id = cpu_to_le16(chap->transaction); 409 data->rescode = NVME_AUTH_DHCHAP_FAILURE_REASON_FAILED; 410 data->rescode_exp = chap->status; 411 412 return size; 413 } 414 415 static int nvme_auth_dhchap_setup_host_response(struct nvme_ctrl *ctrl, 416 struct nvme_dhchap_queue_context *chap) 417 { 418 struct nvme_auth_hmac_ctx hmac; 419 u8 buf[4], *challenge = chap->c1; 420 int ret; 421 422 dev_dbg(ctrl->device, "%s: qid %d host response seq %u transaction %d\n", 423 __func__, chap->qid, chap->s1, chap->transaction); 424 425 if (!chap->transformed_key) { 426 chap->transformed_key = nvme_auth_transform_key(ctrl->host_key, 427 ctrl->opts->host->nqn); 428 if (IS_ERR(chap->transformed_key)) { 429 ret = PTR_ERR(chap->transformed_key); 430 chap->transformed_key = NULL; 431 return ret; 432 } 433 } else { 434 dev_dbg(ctrl->device, "%s: qid %d re-using host response\n", 435 __func__, chap->qid); 436 } 437 438 ret = nvme_auth_hmac_init(&hmac, chap->hash_id, 439 chap->transformed_key->key, 440 chap->transformed_key->len); 441 if (ret) 442 goto out; 443 444 if (chap->dh_tfm) { 445 challenge = kmalloc(chap->hash_len, GFP_KERNEL); 446 if (!challenge) { 447 ret = -ENOMEM; 448 goto out; 449 } 450 ret = nvme_auth_augmented_challenge(chap->hash_id, 451 chap->sess_key, 452 chap->sess_key_len, 453 chap->c1, challenge, 454 chap->hash_len); 455 if (ret) 456 goto out; 457 } 458 459 nvme_auth_hmac_update(&hmac, challenge, chap->hash_len); 460 461 put_unaligned_le32(chap->s1, buf); 462 nvme_auth_hmac_update(&hmac, buf, 4); 463 464 put_unaligned_le16(chap->transaction, buf); 465 nvme_auth_hmac_update(&hmac, buf, 2); 466 467 *buf = chap->sc_c; 468 nvme_auth_hmac_update(&hmac, buf, 1); 469 nvme_auth_hmac_update(&hmac, "HostHost", 8); 470 nvme_auth_hmac_update(&hmac, ctrl->opts->host->nqn, 471 strlen(ctrl->opts->host->nqn)); 472 memset(buf, 0, sizeof(buf)); 473 nvme_auth_hmac_update(&hmac, buf, 1); 474 nvme_auth_hmac_update(&hmac, ctrl->opts->subsysnqn, 475 strlen(ctrl->opts->subsysnqn)); 476 nvme_auth_hmac_final(&hmac, chap->response); 477 ret = 0; 478 out: 479 if (challenge != chap->c1) 480 kfree(challenge); 481 memzero_explicit(&hmac, sizeof(hmac)); 482 return ret; 483 } 484 485 static int nvme_auth_dhchap_setup_ctrl_response(struct nvme_ctrl *ctrl, 486 struct nvme_dhchap_queue_context *chap) 487 { 488 struct nvme_auth_hmac_ctx hmac; 489 struct nvme_dhchap_key *transformed_key; 490 u8 buf[4], *challenge = chap->c2; 491 int ret; 492 493 transformed_key = nvme_auth_transform_key(ctrl->ctrl_key, 494 ctrl->opts->subsysnqn); 495 if (IS_ERR(transformed_key)) { 496 ret = PTR_ERR(transformed_key); 497 return ret; 498 } 499 500 ret = nvme_auth_hmac_init(&hmac, chap->hash_id, transformed_key->key, 501 transformed_key->len); 502 if (ret) { 503 dev_warn(ctrl->device, "qid %d: failed to init hmac, error %d\n", 504 chap->qid, ret); 505 goto out; 506 } 507 508 if (chap->dh_tfm) { 509 challenge = kmalloc(chap->hash_len, GFP_KERNEL); 510 if (!challenge) { 511 ret = -ENOMEM; 512 goto out; 513 } 514 ret = nvme_auth_augmented_challenge(chap->hash_id, 515 chap->sess_key, 516 chap->sess_key_len, 517 chap->c2, challenge, 518 chap->hash_len); 519 if (ret) 520 goto out; 521 } 522 dev_dbg(ctrl->device, "%s: qid %d ctrl response seq %u transaction %d\n", 523 __func__, chap->qid, chap->s2, chap->transaction); 524 dev_dbg(ctrl->device, "%s: qid %d challenge %*ph\n", 525 __func__, chap->qid, (int)chap->hash_len, challenge); 526 dev_dbg(ctrl->device, "%s: qid %d subsysnqn %s\n", 527 __func__, chap->qid, ctrl->opts->subsysnqn); 528 dev_dbg(ctrl->device, "%s: qid %d hostnqn %s\n", 529 __func__, chap->qid, ctrl->opts->host->nqn); 530 531 nvme_auth_hmac_update(&hmac, challenge, chap->hash_len); 532 533 put_unaligned_le32(chap->s2, buf); 534 nvme_auth_hmac_update(&hmac, buf, 4); 535 536 put_unaligned_le16(chap->transaction, buf); 537 nvme_auth_hmac_update(&hmac, buf, 2); 538 539 *buf = chap->sc_c; 540 nvme_auth_hmac_update(&hmac, buf, 1); 541 nvme_auth_hmac_update(&hmac, "Controller", 10); 542 nvme_auth_hmac_update(&hmac, ctrl->opts->subsysnqn, 543 strlen(ctrl->opts->subsysnqn)); 544 memset(buf, 0, 4); 545 nvme_auth_hmac_update(&hmac, buf, 1); 546 nvme_auth_hmac_update(&hmac, ctrl->opts->host->nqn, 547 strlen(ctrl->opts->host->nqn)); 548 nvme_auth_hmac_final(&hmac, chap->response); 549 ret = 0; 550 out: 551 if (challenge != chap->c2) 552 kfree(challenge); 553 memzero_explicit(&hmac, sizeof(hmac)); 554 nvme_auth_free_key(transformed_key); 555 return ret; 556 } 557 558 static int nvme_auth_dhchap_exponential(struct nvme_ctrl *ctrl, 559 struct nvme_dhchap_queue_context *chap) 560 { 561 int ret; 562 563 if (chap->host_key && chap->host_key_len) { 564 dev_dbg(ctrl->device, 565 "qid %d: reusing host key\n", chap->qid); 566 goto gen_sesskey; 567 } 568 ret = nvme_auth_gen_privkey(chap->dh_tfm, chap->dhgroup_id); 569 if (ret < 0) { 570 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 571 return ret; 572 } 573 574 chap->host_key_len = crypto_kpp_maxsize(chap->dh_tfm); 575 576 chap->host_key = kzalloc(chap->host_key_len, GFP_KERNEL); 577 if (!chap->host_key) { 578 chap->host_key_len = 0; 579 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 580 return -ENOMEM; 581 } 582 ret = nvme_auth_gen_pubkey(chap->dh_tfm, 583 chap->host_key, chap->host_key_len); 584 if (ret) { 585 dev_dbg(ctrl->device, 586 "failed to generate public key, error %d\n", ret); 587 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 588 return ret; 589 } 590 591 gen_sesskey: 592 chap->sess_key_len = chap->hash_len; 593 chap->sess_key = kmalloc(chap->sess_key_len, GFP_KERNEL); 594 if (!chap->sess_key) { 595 chap->sess_key_len = 0; 596 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 597 return -ENOMEM; 598 } 599 600 ret = nvme_auth_gen_session_key(chap->dh_tfm, 601 chap->ctrl_key, chap->ctrl_key_len, 602 chap->sess_key, chap->sess_key_len, 603 chap->hash_id); 604 if (ret) { 605 dev_dbg(ctrl->device, 606 "failed to generate session key, error %d\n", ret); 607 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 608 return ret; 609 } 610 dev_dbg(ctrl->device, "session key %*ph\n", 611 (int)chap->sess_key_len, chap->sess_key); 612 return 0; 613 } 614 615 static void nvme_auth_reset_dhchap(struct nvme_dhchap_queue_context *chap) 616 { 617 nvme_auth_free_key(chap->transformed_key); 618 chap->transformed_key = NULL; 619 kfree_sensitive(chap->host_key); 620 chap->host_key = NULL; 621 chap->host_key_len = 0; 622 kfree_sensitive(chap->ctrl_key); 623 chap->ctrl_key = NULL; 624 chap->ctrl_key_len = 0; 625 kfree_sensitive(chap->sess_key); 626 chap->sess_key = NULL; 627 chap->sess_key_len = 0; 628 chap->status = 0; 629 chap->error = 0; 630 chap->s1 = 0; 631 chap->s2 = 0; 632 chap->bi_directional = false; 633 chap->transaction = 0; 634 memset(chap->c1, 0, sizeof(chap->c1)); 635 memset(chap->c2, 0, sizeof(chap->c2)); 636 mempool_free(chap->buf, nvme_chap_buf_pool); 637 chap->buf = NULL; 638 } 639 640 static void nvme_auth_free_dhchap(struct nvme_dhchap_queue_context *chap) 641 { 642 nvme_auth_reset_dhchap(chap); 643 chap->authenticated = false; 644 if (chap->dh_tfm) 645 crypto_free_kpp(chap->dh_tfm); 646 } 647 648 void nvme_auth_revoke_tls_key(struct nvme_ctrl *ctrl) 649 { 650 dev_dbg(ctrl->device, "Wipe generated TLS PSK %08x\n", 651 key_serial(ctrl->opts->tls_key)); 652 key_revoke(ctrl->opts->tls_key); 653 key_put(ctrl->opts->tls_key); 654 ctrl->opts->tls_key = NULL; 655 } 656 EXPORT_SYMBOL_GPL(nvme_auth_revoke_tls_key); 657 658 static int nvme_auth_secure_concat(struct nvme_ctrl *ctrl, 659 struct nvme_dhchap_queue_context *chap) 660 { 661 u8 *psk, *tls_psk; 662 char *digest; 663 struct key *tls_key; 664 size_t psk_len; 665 int ret = 0; 666 667 if (!chap->sess_key) { 668 dev_warn(ctrl->device, 669 "%s: qid %d no session key negotiated\n", 670 __func__, chap->qid); 671 return -ENOKEY; 672 } 673 674 if (chap->qid) { 675 dev_warn(ctrl->device, 676 "qid %d: secure concatenation not supported on I/O queues\n", 677 chap->qid); 678 return -EINVAL; 679 } 680 ret = nvme_auth_generate_psk(chap->hash_id, chap->sess_key, 681 chap->sess_key_len, 682 chap->c1, chap->c2, 683 chap->hash_len, &psk, &psk_len); 684 if (ret) { 685 dev_warn(ctrl->device, 686 "%s: qid %d failed to generate PSK, error %d\n", 687 __func__, chap->qid, ret); 688 return ret; 689 } 690 dev_dbg(ctrl->device, 691 "%s: generated psk %*ph\n", __func__, (int)psk_len, psk); 692 693 ret = nvme_auth_generate_digest(chap->hash_id, psk, psk_len, 694 ctrl->opts->subsysnqn, 695 ctrl->opts->host->nqn, &digest); 696 if (ret) { 697 dev_warn(ctrl->device, 698 "%s: qid %d failed to generate digest, error %d\n", 699 __func__, chap->qid, ret); 700 goto out_free_psk; 701 } 702 dev_dbg(ctrl->device, "%s: generated digest %s\n", 703 __func__, digest); 704 ret = nvme_auth_derive_tls_psk(chap->hash_id, psk, psk_len, 705 digest, &tls_psk); 706 if (ret) { 707 dev_warn(ctrl->device, 708 "%s: qid %d failed to derive TLS psk, error %d\n", 709 __func__, chap->qid, ret); 710 goto out_free_digest; 711 } 712 713 tls_key = nvme_tls_psk_refresh(ctrl->opts->keyring, 714 ctrl->opts->host->nqn, 715 ctrl->opts->subsysnqn, chap->hash_id, 716 tls_psk, psk_len, digest); 717 if (IS_ERR(tls_key)) { 718 ret = PTR_ERR(tls_key); 719 dev_warn(ctrl->device, 720 "%s: qid %d failed to insert generated key, error %d\n", 721 __func__, chap->qid, ret); 722 tls_key = NULL; 723 } 724 kfree_sensitive(tls_psk); 725 if (ctrl->opts->tls_key) 726 nvme_auth_revoke_tls_key(ctrl); 727 ctrl->opts->tls_key = tls_key; 728 out_free_digest: 729 kfree_sensitive(digest); 730 out_free_psk: 731 kfree_sensitive(psk); 732 return ret; 733 } 734 735 static void nvme_queue_auth_work(struct work_struct *work) 736 { 737 struct nvme_dhchap_queue_context *chap = 738 container_of(work, struct nvme_dhchap_queue_context, auth_work); 739 struct nvme_ctrl *ctrl = chap->ctrl; 740 size_t tl; 741 int ret = 0; 742 743 /* 744 * Allocate a large enough buffer for the entire negotiation: 745 * 4k is enough to ffdhe8192. 746 */ 747 chap->buf = mempool_alloc(nvme_chap_buf_pool, GFP_KERNEL); 748 if (!chap->buf) { 749 chap->error = -ENOMEM; 750 return; 751 } 752 753 chap->transaction = ctrl->transaction++; 754 755 /* DH-HMAC-CHAP Step 1: send negotiate */ 756 dev_dbg(ctrl->device, "%s: qid %d send negotiate\n", 757 __func__, chap->qid); 758 ret = nvme_auth_set_dhchap_negotiate_data(ctrl, chap); 759 if (ret < 0) { 760 chap->error = ret; 761 return; 762 } 763 tl = ret; 764 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true); 765 if (ret) { 766 chap->error = ret; 767 return; 768 } 769 770 /* DH-HMAC-CHAP Step 2: receive challenge */ 771 dev_dbg(ctrl->device, "%s: qid %d receive challenge\n", 772 __func__, chap->qid); 773 774 memset(chap->buf, 0, CHAP_BUF_SIZE); 775 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, CHAP_BUF_SIZE, 776 false); 777 if (ret) { 778 dev_warn(ctrl->device, 779 "qid %d failed to receive challenge, %s %d\n", 780 chap->qid, ret < 0 ? "error" : "nvme status", ret); 781 chap->error = ret; 782 return; 783 } 784 ret = nvme_auth_receive_validate(ctrl, chap->qid, chap->buf, chap->transaction, 785 NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE); 786 if (ret) { 787 chap->status = ret; 788 chap->error = -EKEYREJECTED; 789 return; 790 } 791 792 ret = nvme_auth_process_dhchap_challenge(ctrl, chap); 793 if (ret) { 794 /* Invalid challenge parameters */ 795 chap->error = ret; 796 goto fail2; 797 } 798 799 if (chap->ctrl_key_len) { 800 dev_dbg(ctrl->device, 801 "%s: qid %d DH exponential\n", 802 __func__, chap->qid); 803 ret = nvme_auth_dhchap_exponential(ctrl, chap); 804 if (ret) { 805 chap->error = ret; 806 goto fail2; 807 } 808 } 809 810 dev_dbg(ctrl->device, "%s: qid %d host response\n", 811 __func__, chap->qid); 812 mutex_lock(&ctrl->dhchap_auth_mutex); 813 ret = nvme_auth_dhchap_setup_host_response(ctrl, chap); 814 mutex_unlock(&ctrl->dhchap_auth_mutex); 815 if (ret) { 816 chap->error = ret; 817 goto fail2; 818 } 819 820 /* DH-HMAC-CHAP Step 3: send reply */ 821 dev_dbg(ctrl->device, "%s: qid %d send reply\n", 822 __func__, chap->qid); 823 ret = nvme_auth_set_dhchap_reply_data(ctrl, chap); 824 if (ret < 0) { 825 chap->error = ret; 826 goto fail2; 827 } 828 829 tl = ret; 830 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true); 831 if (ret) { 832 chap->error = ret; 833 goto fail2; 834 } 835 836 /* DH-HMAC-CHAP Step 4: receive success1 */ 837 dev_dbg(ctrl->device, "%s: qid %d receive success1\n", 838 __func__, chap->qid); 839 840 memset(chap->buf, 0, CHAP_BUF_SIZE); 841 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, CHAP_BUF_SIZE, 842 false); 843 if (ret) { 844 dev_warn(ctrl->device, 845 "qid %d failed to receive success1, %s %d\n", 846 chap->qid, ret < 0 ? "error" : "nvme status", ret); 847 chap->error = ret; 848 return; 849 } 850 ret = nvme_auth_receive_validate(ctrl, chap->qid, 851 chap->buf, chap->transaction, 852 NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1); 853 if (ret) { 854 chap->status = ret; 855 chap->error = -EKEYREJECTED; 856 return; 857 } 858 859 mutex_lock(&ctrl->dhchap_auth_mutex); 860 if (ctrl->ctrl_key) { 861 dev_dbg(ctrl->device, 862 "%s: qid %d controller response\n", 863 __func__, chap->qid); 864 ret = nvme_auth_dhchap_setup_ctrl_response(ctrl, chap); 865 if (ret) { 866 mutex_unlock(&ctrl->dhchap_auth_mutex); 867 chap->error = ret; 868 goto fail2; 869 } 870 } 871 mutex_unlock(&ctrl->dhchap_auth_mutex); 872 873 ret = nvme_auth_process_dhchap_success1(ctrl, chap); 874 if (ret) { 875 /* Controller authentication failed */ 876 chap->error = -EKEYREJECTED; 877 goto fail2; 878 } 879 880 if (chap->bi_directional) { 881 /* DH-HMAC-CHAP Step 5: send success2 */ 882 dev_dbg(ctrl->device, "%s: qid %d send success2\n", 883 __func__, chap->qid); 884 tl = nvme_auth_set_dhchap_success2_data(ctrl, chap); 885 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true); 886 if (ret) 887 chap->error = ret; 888 } 889 if (!ret) { 890 chap->error = 0; 891 chap->authenticated = true; 892 if (ctrl->opts->concat && 893 (ret = nvme_auth_secure_concat(ctrl, chap))) { 894 dev_warn(ctrl->device, 895 "%s: qid %d failed to enable secure concatenation\n", 896 __func__, chap->qid); 897 chap->error = ret; 898 chap->authenticated = false; 899 } 900 return; 901 } 902 903 fail2: 904 if (chap->status == 0) 905 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 906 dev_dbg(ctrl->device, "%s: qid %d send failure2, status %x\n", 907 __func__, chap->qid, chap->status); 908 tl = nvme_auth_set_dhchap_failure2_data(ctrl, chap); 909 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true); 910 /* 911 * only update error if send failure2 failed and no other 912 * error had been set during authentication. 913 */ 914 if (ret && !chap->error) 915 chap->error = ret; 916 } 917 918 int nvme_auth_negotiate(struct nvme_ctrl *ctrl, int qid) 919 { 920 struct nvme_dhchap_queue_context *chap; 921 922 if (!ctrl->host_key) { 923 dev_warn(ctrl->device, "qid %d: no key\n", qid); 924 return -ENOKEY; 925 } 926 927 if (ctrl->opts->dhchap_ctrl_secret && !ctrl->ctrl_key) { 928 dev_warn(ctrl->device, "qid %d: invalid ctrl key\n", qid); 929 return -ENOKEY; 930 } 931 932 chap = &ctrl->dhchap_ctxs[qid]; 933 cancel_work_sync(&chap->auth_work); 934 queue_work(nvme_auth_wq, &chap->auth_work); 935 return 0; 936 } 937 EXPORT_SYMBOL_GPL(nvme_auth_negotiate); 938 939 int nvme_auth_wait(struct nvme_ctrl *ctrl, int qid) 940 { 941 struct nvme_dhchap_queue_context *chap; 942 int ret; 943 944 chap = &ctrl->dhchap_ctxs[qid]; 945 flush_work(&chap->auth_work); 946 ret = chap->error; 947 /* clear sensitive info */ 948 nvme_auth_reset_dhchap(chap); 949 return ret; 950 } 951 EXPORT_SYMBOL_GPL(nvme_auth_wait); 952 953 static void nvme_ctrl_auth_work(struct work_struct *work) 954 { 955 struct nvme_ctrl *ctrl = 956 container_of(work, struct nvme_ctrl, dhchap_auth_work); 957 int ret, q; 958 959 /* 960 * If the ctrl is no connected, bail as reconnect will handle 961 * authentication. 962 */ 963 if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE) 964 return; 965 966 /* Authenticate admin queue first */ 967 ret = nvme_auth_negotiate(ctrl, 0); 968 if (ret) { 969 dev_warn(ctrl->device, 970 "qid 0: error %d setting up authentication\n", ret); 971 return; 972 } 973 ret = nvme_auth_wait(ctrl, 0); 974 if (ret) { 975 dev_warn(ctrl->device, 976 "qid 0: authentication failed\n"); 977 return; 978 } 979 /* 980 * Only run authentication on the admin queue for secure concatenation. 981 */ 982 if (ctrl->opts->concat) 983 return; 984 985 for (q = 1; q < ctrl->queue_count; q++) { 986 struct nvme_dhchap_queue_context *chap = 987 &ctrl->dhchap_ctxs[q]; 988 /* 989 * Skip re-authentication if the queue had 990 * not been authenticated initially. 991 */ 992 if (!chap->authenticated) 993 continue; 994 cancel_work_sync(&chap->auth_work); 995 queue_work(nvme_auth_wq, &chap->auth_work); 996 } 997 998 /* 999 * Failure is a soft-state; credentials remain valid until 1000 * the controller terminates the connection. 1001 */ 1002 for (q = 1; q < ctrl->queue_count; q++) { 1003 struct nvme_dhchap_queue_context *chap = 1004 &ctrl->dhchap_ctxs[q]; 1005 if (!chap->authenticated) 1006 continue; 1007 flush_work(&chap->auth_work); 1008 ret = chap->error; 1009 nvme_auth_reset_dhchap(chap); 1010 if (ret) 1011 dev_warn(ctrl->device, 1012 "qid %d: authentication failed\n", q); 1013 } 1014 } 1015 1016 int nvme_auth_init_ctrl(struct nvme_ctrl *ctrl) 1017 { 1018 struct nvme_dhchap_queue_context *chap; 1019 int i, ret; 1020 1021 mutex_init(&ctrl->dhchap_auth_mutex); 1022 INIT_WORK(&ctrl->dhchap_auth_work, nvme_ctrl_auth_work); 1023 if (!ctrl->opts) 1024 return 0; 1025 ret = nvme_auth_parse_key(ctrl->opts->dhchap_secret, &ctrl->host_key); 1026 if (ret) 1027 return ret; 1028 ret = nvme_auth_parse_key(ctrl->opts->dhchap_ctrl_secret, 1029 &ctrl->ctrl_key); 1030 if (ret) 1031 goto err_free_dhchap_secret; 1032 1033 if (!ctrl->opts->dhchap_secret && !ctrl->opts->dhchap_ctrl_secret) 1034 return 0; 1035 1036 ctrl->dhchap_ctxs = kvzalloc_objs(*chap, ctrl_max_dhchaps(ctrl)); 1037 if (!ctrl->dhchap_ctxs) { 1038 ret = -ENOMEM; 1039 goto err_free_dhchap_ctrl_secret; 1040 } 1041 1042 for (i = 0; i < ctrl_max_dhchaps(ctrl); i++) { 1043 chap = &ctrl->dhchap_ctxs[i]; 1044 chap->qid = i; 1045 chap->ctrl = ctrl; 1046 chap->authenticated = false; 1047 INIT_WORK(&chap->auth_work, nvme_queue_auth_work); 1048 } 1049 1050 return 0; 1051 err_free_dhchap_ctrl_secret: 1052 nvme_auth_free_key(ctrl->ctrl_key); 1053 ctrl->ctrl_key = NULL; 1054 err_free_dhchap_secret: 1055 nvme_auth_free_key(ctrl->host_key); 1056 ctrl->host_key = NULL; 1057 return ret; 1058 } 1059 EXPORT_SYMBOL_GPL(nvme_auth_init_ctrl); 1060 1061 void nvme_auth_stop(struct nvme_ctrl *ctrl) 1062 { 1063 cancel_work_sync(&ctrl->dhchap_auth_work); 1064 } 1065 EXPORT_SYMBOL_GPL(nvme_auth_stop); 1066 1067 void nvme_auth_free(struct nvme_ctrl *ctrl) 1068 { 1069 int i; 1070 1071 if (ctrl->dhchap_ctxs) { 1072 for (i = 0; i < ctrl_max_dhchaps(ctrl); i++) 1073 nvme_auth_free_dhchap(&ctrl->dhchap_ctxs[i]); 1074 kvfree(ctrl->dhchap_ctxs); 1075 } 1076 if (ctrl->host_key) { 1077 nvme_auth_free_key(ctrl->host_key); 1078 ctrl->host_key = NULL; 1079 } 1080 if (ctrl->ctrl_key) { 1081 nvme_auth_free_key(ctrl->ctrl_key); 1082 ctrl->ctrl_key = NULL; 1083 } 1084 } 1085 EXPORT_SYMBOL_GPL(nvme_auth_free); 1086 1087 int __init nvme_init_auth(void) 1088 { 1089 nvme_auth_wq = alloc_workqueue("nvme-auth-wq", 1090 WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0); 1091 if (!nvme_auth_wq) 1092 return -ENOMEM; 1093 1094 nvme_chap_buf_cache = kmem_cache_create("nvme-chap-buf-cache", 1095 CHAP_BUF_SIZE, 0, SLAB_HWCACHE_ALIGN, NULL); 1096 if (!nvme_chap_buf_cache) 1097 goto err_destroy_workqueue; 1098 1099 nvme_chap_buf_pool = mempool_create(16, mempool_alloc_slab, 1100 mempool_free_slab, nvme_chap_buf_cache); 1101 if (!nvme_chap_buf_pool) 1102 goto err_destroy_chap_buf_cache; 1103 1104 return 0; 1105 err_destroy_chap_buf_cache: 1106 kmem_cache_destroy(nvme_chap_buf_cache); 1107 err_destroy_workqueue: 1108 destroy_workqueue(nvme_auth_wq); 1109 return -ENOMEM; 1110 } 1111 1112 void __exit nvme_exit_auth(void) 1113 { 1114 mempool_destroy(nvme_chap_buf_pool); 1115 kmem_cache_destroy(nvme_chap_buf_cache); 1116 destroy_workqueue(nvme_auth_wq); 1117 } 1118