1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2020 Hannes Reinecke, SUSE Linux 4 */ 5 6 #include <linux/crc32.h> 7 #include <linux/base64.h> 8 #include <linux/prandom.h> 9 #include <linux/unaligned.h> 10 #include <crypto/hash.h> 11 #include <crypto/dh.h> 12 #include "nvme.h" 13 #include "fabrics.h" 14 #include <linux/nvme-auth.h> 15 #include <linux/nvme-keyring.h> 16 17 #define CHAP_BUF_SIZE 4096 18 static struct kmem_cache *nvme_chap_buf_cache; 19 static mempool_t *nvme_chap_buf_pool; 20 21 struct nvme_dhchap_queue_context { 22 struct list_head entry; 23 struct work_struct auth_work; 24 struct nvme_ctrl *ctrl; 25 struct crypto_shash *shash_tfm; 26 struct crypto_kpp *dh_tfm; 27 struct nvme_dhchap_key *transformed_key; 28 void *buf; 29 int qid; 30 int error; 31 u32 s1; 32 u32 s2; 33 bool bi_directional; 34 bool authenticated; 35 u16 transaction; 36 u8 status; 37 u8 dhgroup_id; 38 u8 hash_id; 39 size_t hash_len; 40 u8 c1[64]; 41 u8 c2[64]; 42 u8 response[64]; 43 u8 *ctrl_key; 44 u8 *host_key; 45 u8 *sess_key; 46 int ctrl_key_len; 47 int host_key_len; 48 int sess_key_len; 49 }; 50 51 static struct workqueue_struct *nvme_auth_wq; 52 53 static inline int ctrl_max_dhchaps(struct nvme_ctrl *ctrl) 54 { 55 return ctrl->opts->nr_io_queues + ctrl->opts->nr_write_queues + 56 ctrl->opts->nr_poll_queues + 1; 57 } 58 59 static int nvme_auth_submit(struct nvme_ctrl *ctrl, int qid, 60 void *data, size_t data_len, bool auth_send) 61 { 62 struct nvme_command cmd = {}; 63 nvme_submit_flags_t flags = NVME_SUBMIT_RETRY; 64 struct request_queue *q = ctrl->fabrics_q; 65 int ret; 66 67 if (qid != 0) { 68 flags |= NVME_SUBMIT_NOWAIT | NVME_SUBMIT_RESERVED; 69 q = ctrl->connect_q; 70 } 71 72 cmd.auth_common.opcode = nvme_fabrics_command; 73 cmd.auth_common.secp = NVME_AUTH_DHCHAP_PROTOCOL_IDENTIFIER; 74 cmd.auth_common.spsp0 = 0x01; 75 cmd.auth_common.spsp1 = 0x01; 76 if (auth_send) { 77 cmd.auth_send.fctype = nvme_fabrics_type_auth_send; 78 cmd.auth_send.tl = cpu_to_le32(data_len); 79 } else { 80 cmd.auth_receive.fctype = nvme_fabrics_type_auth_receive; 81 cmd.auth_receive.al = cpu_to_le32(data_len); 82 } 83 84 ret = __nvme_submit_sync_cmd(q, &cmd, NULL, data, data_len, 85 qid == 0 ? NVME_QID_ANY : qid, flags); 86 if (ret > 0) 87 dev_warn(ctrl->device, 88 "qid %d auth_send failed with status %d\n", qid, ret); 89 else if (ret < 0) 90 dev_err(ctrl->device, 91 "qid %d auth_send failed with error %d\n", qid, ret); 92 return ret; 93 } 94 95 static int nvme_auth_receive_validate(struct nvme_ctrl *ctrl, int qid, 96 struct nvmf_auth_dhchap_failure_data *data, 97 u16 transaction, u8 expected_msg) 98 { 99 dev_dbg(ctrl->device, "%s: qid %d auth_type %d auth_id %x\n", 100 __func__, qid, data->auth_type, data->auth_id); 101 102 if (data->auth_type == NVME_AUTH_COMMON_MESSAGES && 103 data->auth_id == NVME_AUTH_DHCHAP_MESSAGE_FAILURE1) { 104 return data->rescode_exp; 105 } 106 if (data->auth_type != NVME_AUTH_DHCHAP_MESSAGES || 107 data->auth_id != expected_msg) { 108 dev_warn(ctrl->device, 109 "qid %d invalid message %02x/%02x\n", 110 qid, data->auth_type, data->auth_id); 111 return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE; 112 } 113 if (le16_to_cpu(data->t_id) != transaction) { 114 dev_warn(ctrl->device, 115 "qid %d invalid transaction ID %d\n", 116 qid, le16_to_cpu(data->t_id)); 117 return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE; 118 } 119 return 0; 120 } 121 122 static int nvme_auth_set_dhchap_negotiate_data(struct nvme_ctrl *ctrl, 123 struct nvme_dhchap_queue_context *chap) 124 { 125 struct nvmf_auth_dhchap_negotiate_data *data = chap->buf; 126 size_t size = sizeof(*data) + sizeof(union nvmf_auth_protocol); 127 128 if (size > CHAP_BUF_SIZE) { 129 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 130 return -EINVAL; 131 } 132 memset((u8 *)chap->buf, 0, size); 133 data->auth_type = NVME_AUTH_COMMON_MESSAGES; 134 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE; 135 data->t_id = cpu_to_le16(chap->transaction); 136 if (ctrl->opts->concat && chap->qid == 0) { 137 if (ctrl->opts->tls_key) 138 data->sc_c = NVME_AUTH_SECP_REPLACETLSPSK; 139 else 140 data->sc_c = NVME_AUTH_SECP_NEWTLSPSK; 141 } else 142 data->sc_c = NVME_AUTH_SECP_NOSC; 143 data->napd = 1; 144 data->auth_protocol[0].dhchap.authid = NVME_AUTH_DHCHAP_AUTH_ID; 145 data->auth_protocol[0].dhchap.halen = 3; 146 data->auth_protocol[0].dhchap.dhlen = 6; 147 data->auth_protocol[0].dhchap.idlist[0] = NVME_AUTH_HASH_SHA256; 148 data->auth_protocol[0].dhchap.idlist[1] = NVME_AUTH_HASH_SHA384; 149 data->auth_protocol[0].dhchap.idlist[2] = NVME_AUTH_HASH_SHA512; 150 data->auth_protocol[0].dhchap.idlist[30] = NVME_AUTH_DHGROUP_NULL; 151 data->auth_protocol[0].dhchap.idlist[31] = NVME_AUTH_DHGROUP_2048; 152 data->auth_protocol[0].dhchap.idlist[32] = NVME_AUTH_DHGROUP_3072; 153 data->auth_protocol[0].dhchap.idlist[33] = NVME_AUTH_DHGROUP_4096; 154 data->auth_protocol[0].dhchap.idlist[34] = NVME_AUTH_DHGROUP_6144; 155 data->auth_protocol[0].dhchap.idlist[35] = NVME_AUTH_DHGROUP_8192; 156 157 return size; 158 } 159 160 static int nvme_auth_process_dhchap_challenge(struct nvme_ctrl *ctrl, 161 struct nvme_dhchap_queue_context *chap) 162 { 163 struct nvmf_auth_dhchap_challenge_data *data = chap->buf; 164 u16 dhvlen = le16_to_cpu(data->dhvlen); 165 size_t size = sizeof(*data) + data->hl + dhvlen; 166 const char *gid_name = nvme_auth_dhgroup_name(data->dhgid); 167 const char *hmac_name, *kpp_name; 168 169 if (size > CHAP_BUF_SIZE) { 170 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 171 return -EINVAL; 172 } 173 174 hmac_name = nvme_auth_hmac_name(data->hashid); 175 if (!hmac_name) { 176 dev_warn(ctrl->device, 177 "qid %d: invalid HASH ID %d\n", 178 chap->qid, data->hashid); 179 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE; 180 return -EPROTO; 181 } 182 183 if (chap->hash_id == data->hashid && chap->shash_tfm && 184 !strcmp(crypto_shash_alg_name(chap->shash_tfm), hmac_name) && 185 crypto_shash_digestsize(chap->shash_tfm) == data->hl) { 186 dev_dbg(ctrl->device, 187 "qid %d: reuse existing hash %s\n", 188 chap->qid, hmac_name); 189 goto select_kpp; 190 } 191 192 /* Reset if hash cannot be reused */ 193 if (chap->shash_tfm) { 194 crypto_free_shash(chap->shash_tfm); 195 chap->hash_id = 0; 196 chap->hash_len = 0; 197 } 198 chap->shash_tfm = crypto_alloc_shash(hmac_name, 0, 199 CRYPTO_ALG_ALLOCATES_MEMORY); 200 if (IS_ERR(chap->shash_tfm)) { 201 dev_warn(ctrl->device, 202 "qid %d: failed to allocate hash %s, error %ld\n", 203 chap->qid, hmac_name, PTR_ERR(chap->shash_tfm)); 204 chap->shash_tfm = NULL; 205 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 206 return -ENOMEM; 207 } 208 209 if (crypto_shash_digestsize(chap->shash_tfm) != data->hl) { 210 dev_warn(ctrl->device, 211 "qid %d: invalid hash length %d\n", 212 chap->qid, data->hl); 213 crypto_free_shash(chap->shash_tfm); 214 chap->shash_tfm = NULL; 215 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE; 216 return -EPROTO; 217 } 218 219 chap->hash_id = data->hashid; 220 chap->hash_len = data->hl; 221 dev_dbg(ctrl->device, "qid %d: selected hash %s\n", 222 chap->qid, hmac_name); 223 224 select_kpp: 225 kpp_name = nvme_auth_dhgroup_kpp(data->dhgid); 226 if (!kpp_name) { 227 dev_warn(ctrl->device, 228 "qid %d: invalid DH group id %d\n", 229 chap->qid, data->dhgid); 230 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE; 231 /* Leave previous dh_tfm intact */ 232 return -EPROTO; 233 } 234 235 if (chap->dhgroup_id == data->dhgid && 236 (data->dhgid == NVME_AUTH_DHGROUP_NULL || chap->dh_tfm)) { 237 dev_dbg(ctrl->device, 238 "qid %d: reuse existing DH group %s\n", 239 chap->qid, gid_name); 240 goto skip_kpp; 241 } 242 243 /* Reset dh_tfm if it can't be reused */ 244 if (chap->dh_tfm) { 245 crypto_free_kpp(chap->dh_tfm); 246 chap->dh_tfm = NULL; 247 } 248 249 if (data->dhgid != NVME_AUTH_DHGROUP_NULL) { 250 if (dhvlen == 0) { 251 dev_warn(ctrl->device, 252 "qid %d: empty DH value\n", 253 chap->qid); 254 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE; 255 return -EPROTO; 256 } 257 258 chap->dh_tfm = crypto_alloc_kpp(kpp_name, 0, 0); 259 if (IS_ERR(chap->dh_tfm)) { 260 int ret = PTR_ERR(chap->dh_tfm); 261 262 dev_warn(ctrl->device, 263 "qid %d: error %d initializing DH group %s\n", 264 chap->qid, ret, gid_name); 265 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE; 266 chap->dh_tfm = NULL; 267 return ret; 268 } 269 dev_dbg(ctrl->device, "qid %d: selected DH group %s\n", 270 chap->qid, gid_name); 271 } else if (dhvlen != 0) { 272 dev_warn(ctrl->device, 273 "qid %d: invalid DH value for NULL DH\n", 274 chap->qid); 275 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 276 return -EPROTO; 277 } 278 chap->dhgroup_id = data->dhgid; 279 280 skip_kpp: 281 chap->s1 = le32_to_cpu(data->seqnum); 282 memcpy(chap->c1, data->cval, chap->hash_len); 283 if (dhvlen) { 284 chap->ctrl_key = kmalloc(dhvlen, GFP_KERNEL); 285 if (!chap->ctrl_key) { 286 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 287 return -ENOMEM; 288 } 289 chap->ctrl_key_len = dhvlen; 290 memcpy(chap->ctrl_key, data->cval + chap->hash_len, 291 dhvlen); 292 dev_dbg(ctrl->device, "ctrl public key %*ph\n", 293 (int)chap->ctrl_key_len, chap->ctrl_key); 294 } 295 296 return 0; 297 } 298 299 static int nvme_auth_set_dhchap_reply_data(struct nvme_ctrl *ctrl, 300 struct nvme_dhchap_queue_context *chap) 301 { 302 struct nvmf_auth_dhchap_reply_data *data = chap->buf; 303 size_t size = sizeof(*data); 304 305 size += 2 * chap->hash_len; 306 307 if (chap->host_key_len) 308 size += chap->host_key_len; 309 310 if (size > CHAP_BUF_SIZE) { 311 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 312 return -EINVAL; 313 } 314 315 memset(chap->buf, 0, size); 316 data->auth_type = NVME_AUTH_DHCHAP_MESSAGES; 317 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_REPLY; 318 data->t_id = cpu_to_le16(chap->transaction); 319 data->hl = chap->hash_len; 320 data->dhvlen = cpu_to_le16(chap->host_key_len); 321 memcpy(data->rval, chap->response, chap->hash_len); 322 if (ctrl->ctrl_key) 323 chap->bi_directional = true; 324 if (ctrl->ctrl_key || ctrl->opts->concat) { 325 get_random_bytes(chap->c2, chap->hash_len); 326 data->cvalid = 1; 327 memcpy(data->rval + chap->hash_len, chap->c2, 328 chap->hash_len); 329 dev_dbg(ctrl->device, "%s: qid %d ctrl challenge %*ph\n", 330 __func__, chap->qid, (int)chap->hash_len, chap->c2); 331 } else { 332 memset(chap->c2, 0, chap->hash_len); 333 } 334 if (ctrl->opts->concat) 335 chap->s2 = 0; 336 else 337 chap->s2 = nvme_auth_get_seqnum(); 338 data->seqnum = cpu_to_le32(chap->s2); 339 if (chap->host_key_len) { 340 dev_dbg(ctrl->device, "%s: qid %d host public key %*ph\n", 341 __func__, chap->qid, 342 chap->host_key_len, chap->host_key); 343 memcpy(data->rval + 2 * chap->hash_len, chap->host_key, 344 chap->host_key_len); 345 } 346 347 return size; 348 } 349 350 static int nvme_auth_process_dhchap_success1(struct nvme_ctrl *ctrl, 351 struct nvme_dhchap_queue_context *chap) 352 { 353 struct nvmf_auth_dhchap_success1_data *data = chap->buf; 354 size_t size = sizeof(*data) + chap->hash_len; 355 356 if (size > CHAP_BUF_SIZE) { 357 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 358 return -EINVAL; 359 } 360 361 if (data->hl != chap->hash_len) { 362 dev_warn(ctrl->device, 363 "qid %d: invalid hash length %u\n", 364 chap->qid, data->hl); 365 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE; 366 return -EPROTO; 367 } 368 369 /* Just print out information for the admin queue */ 370 if (chap->qid == 0) 371 dev_info(ctrl->device, 372 "qid 0: authenticated with hash %s dhgroup %s\n", 373 nvme_auth_hmac_name(chap->hash_id), 374 nvme_auth_dhgroup_name(chap->dhgroup_id)); 375 376 if (!data->rvalid) 377 return 0; 378 379 /* Validate controller response */ 380 if (memcmp(chap->response, data->rval, data->hl)) { 381 dev_dbg(ctrl->device, "%s: qid %d ctrl response %*ph\n", 382 __func__, chap->qid, (int)chap->hash_len, data->rval); 383 dev_dbg(ctrl->device, "%s: qid %d host response %*ph\n", 384 __func__, chap->qid, (int)chap->hash_len, 385 chap->response); 386 dev_warn(ctrl->device, 387 "qid %d: controller authentication failed\n", 388 chap->qid); 389 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 390 return -ECONNREFUSED; 391 } 392 393 /* Just print out information for the admin queue */ 394 if (chap->qid == 0) 395 dev_info(ctrl->device, 396 "qid 0: controller authenticated\n"); 397 return 0; 398 } 399 400 static int nvme_auth_set_dhchap_success2_data(struct nvme_ctrl *ctrl, 401 struct nvme_dhchap_queue_context *chap) 402 { 403 struct nvmf_auth_dhchap_success2_data *data = chap->buf; 404 size_t size = sizeof(*data); 405 406 memset(chap->buf, 0, size); 407 data->auth_type = NVME_AUTH_DHCHAP_MESSAGES; 408 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2; 409 data->t_id = cpu_to_le16(chap->transaction); 410 411 return size; 412 } 413 414 static int nvme_auth_set_dhchap_failure2_data(struct nvme_ctrl *ctrl, 415 struct nvme_dhchap_queue_context *chap) 416 { 417 struct nvmf_auth_dhchap_failure_data *data = chap->buf; 418 size_t size = sizeof(*data); 419 420 memset(chap->buf, 0, size); 421 data->auth_type = NVME_AUTH_COMMON_MESSAGES; 422 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_FAILURE2; 423 data->t_id = cpu_to_le16(chap->transaction); 424 data->rescode = NVME_AUTH_DHCHAP_FAILURE_REASON_FAILED; 425 data->rescode_exp = chap->status; 426 427 return size; 428 } 429 430 static int nvme_auth_dhchap_setup_host_response(struct nvme_ctrl *ctrl, 431 struct nvme_dhchap_queue_context *chap) 432 { 433 SHASH_DESC_ON_STACK(shash, chap->shash_tfm); 434 u8 buf[4], *challenge = chap->c1; 435 int ret; 436 437 dev_dbg(ctrl->device, "%s: qid %d host response seq %u transaction %d\n", 438 __func__, chap->qid, chap->s1, chap->transaction); 439 440 if (!chap->transformed_key) { 441 chap->transformed_key = nvme_auth_transform_key(ctrl->host_key, 442 ctrl->opts->host->nqn); 443 if (IS_ERR(chap->transformed_key)) { 444 ret = PTR_ERR(chap->transformed_key); 445 chap->transformed_key = NULL; 446 return ret; 447 } 448 } else { 449 dev_dbg(ctrl->device, "%s: qid %d re-using host response\n", 450 __func__, chap->qid); 451 } 452 453 ret = crypto_shash_setkey(chap->shash_tfm, 454 chap->transformed_key->key, chap->transformed_key->len); 455 if (ret) { 456 dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n", 457 chap->qid, ret); 458 goto out; 459 } 460 461 if (chap->dh_tfm) { 462 challenge = kmalloc(chap->hash_len, GFP_KERNEL); 463 if (!challenge) { 464 ret = -ENOMEM; 465 goto out; 466 } 467 ret = nvme_auth_augmented_challenge(chap->hash_id, 468 chap->sess_key, 469 chap->sess_key_len, 470 chap->c1, challenge, 471 chap->hash_len); 472 if (ret) 473 goto out; 474 } 475 476 shash->tfm = chap->shash_tfm; 477 ret = crypto_shash_init(shash); 478 if (ret) 479 goto out; 480 ret = crypto_shash_update(shash, challenge, chap->hash_len); 481 if (ret) 482 goto out; 483 put_unaligned_le32(chap->s1, buf); 484 ret = crypto_shash_update(shash, buf, 4); 485 if (ret) 486 goto out; 487 put_unaligned_le16(chap->transaction, buf); 488 ret = crypto_shash_update(shash, buf, 2); 489 if (ret) 490 goto out; 491 memset(buf, 0, sizeof(buf)); 492 ret = crypto_shash_update(shash, buf, 1); 493 if (ret) 494 goto out; 495 ret = crypto_shash_update(shash, "HostHost", 8); 496 if (ret) 497 goto out; 498 ret = crypto_shash_update(shash, ctrl->opts->host->nqn, 499 strlen(ctrl->opts->host->nqn)); 500 if (ret) 501 goto out; 502 ret = crypto_shash_update(shash, buf, 1); 503 if (ret) 504 goto out; 505 ret = crypto_shash_update(shash, ctrl->opts->subsysnqn, 506 strlen(ctrl->opts->subsysnqn)); 507 if (ret) 508 goto out; 509 ret = crypto_shash_final(shash, chap->response); 510 out: 511 if (challenge != chap->c1) 512 kfree(challenge); 513 return ret; 514 } 515 516 static int nvme_auth_dhchap_setup_ctrl_response(struct nvme_ctrl *ctrl, 517 struct nvme_dhchap_queue_context *chap) 518 { 519 SHASH_DESC_ON_STACK(shash, chap->shash_tfm); 520 struct nvme_dhchap_key *transformed_key; 521 u8 buf[4], *challenge = chap->c2; 522 int ret; 523 524 transformed_key = nvme_auth_transform_key(ctrl->ctrl_key, 525 ctrl->opts->subsysnqn); 526 if (IS_ERR(transformed_key)) { 527 ret = PTR_ERR(transformed_key); 528 return ret; 529 } 530 531 ret = crypto_shash_setkey(chap->shash_tfm, 532 transformed_key->key, transformed_key->len); 533 if (ret) { 534 dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n", 535 chap->qid, ret); 536 goto out; 537 } 538 539 if (chap->dh_tfm) { 540 challenge = kmalloc(chap->hash_len, GFP_KERNEL); 541 if (!challenge) { 542 ret = -ENOMEM; 543 goto out; 544 } 545 ret = nvme_auth_augmented_challenge(chap->hash_id, 546 chap->sess_key, 547 chap->sess_key_len, 548 chap->c2, challenge, 549 chap->hash_len); 550 if (ret) 551 goto out; 552 } 553 dev_dbg(ctrl->device, "%s: qid %d ctrl response seq %u transaction %d\n", 554 __func__, chap->qid, chap->s2, chap->transaction); 555 dev_dbg(ctrl->device, "%s: qid %d challenge %*ph\n", 556 __func__, chap->qid, (int)chap->hash_len, challenge); 557 dev_dbg(ctrl->device, "%s: qid %d subsysnqn %s\n", 558 __func__, chap->qid, ctrl->opts->subsysnqn); 559 dev_dbg(ctrl->device, "%s: qid %d hostnqn %s\n", 560 __func__, chap->qid, ctrl->opts->host->nqn); 561 shash->tfm = chap->shash_tfm; 562 ret = crypto_shash_init(shash); 563 if (ret) 564 goto out; 565 ret = crypto_shash_update(shash, challenge, chap->hash_len); 566 if (ret) 567 goto out; 568 put_unaligned_le32(chap->s2, buf); 569 ret = crypto_shash_update(shash, buf, 4); 570 if (ret) 571 goto out; 572 put_unaligned_le16(chap->transaction, buf); 573 ret = crypto_shash_update(shash, buf, 2); 574 if (ret) 575 goto out; 576 memset(buf, 0, 4); 577 ret = crypto_shash_update(shash, buf, 1); 578 if (ret) 579 goto out; 580 ret = crypto_shash_update(shash, "Controller", 10); 581 if (ret) 582 goto out; 583 ret = crypto_shash_update(shash, ctrl->opts->subsysnqn, 584 strlen(ctrl->opts->subsysnqn)); 585 if (ret) 586 goto out; 587 ret = crypto_shash_update(shash, buf, 1); 588 if (ret) 589 goto out; 590 ret = crypto_shash_update(shash, ctrl->opts->host->nqn, 591 strlen(ctrl->opts->host->nqn)); 592 if (ret) 593 goto out; 594 ret = crypto_shash_final(shash, chap->response); 595 out: 596 if (challenge != chap->c2) 597 kfree(challenge); 598 nvme_auth_free_key(transformed_key); 599 return ret; 600 } 601 602 static int nvme_auth_dhchap_exponential(struct nvme_ctrl *ctrl, 603 struct nvme_dhchap_queue_context *chap) 604 { 605 int ret; 606 607 if (chap->host_key && chap->host_key_len) { 608 dev_dbg(ctrl->device, 609 "qid %d: reusing host key\n", chap->qid); 610 goto gen_sesskey; 611 } 612 ret = nvme_auth_gen_privkey(chap->dh_tfm, chap->dhgroup_id); 613 if (ret < 0) { 614 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 615 return ret; 616 } 617 618 chap->host_key_len = crypto_kpp_maxsize(chap->dh_tfm); 619 620 chap->host_key = kzalloc(chap->host_key_len, GFP_KERNEL); 621 if (!chap->host_key) { 622 chap->host_key_len = 0; 623 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 624 return -ENOMEM; 625 } 626 ret = nvme_auth_gen_pubkey(chap->dh_tfm, 627 chap->host_key, chap->host_key_len); 628 if (ret) { 629 dev_dbg(ctrl->device, 630 "failed to generate public key, error %d\n", ret); 631 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 632 return ret; 633 } 634 635 gen_sesskey: 636 chap->sess_key_len = chap->host_key_len; 637 chap->sess_key = kmalloc(chap->sess_key_len, GFP_KERNEL); 638 if (!chap->sess_key) { 639 chap->sess_key_len = 0; 640 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 641 return -ENOMEM; 642 } 643 644 ret = nvme_auth_gen_shared_secret(chap->dh_tfm, 645 chap->ctrl_key, chap->ctrl_key_len, 646 chap->sess_key, chap->sess_key_len); 647 if (ret) { 648 dev_dbg(ctrl->device, 649 "failed to generate shared secret, error %d\n", ret); 650 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 651 return ret; 652 } 653 dev_dbg(ctrl->device, "shared secret %*ph\n", 654 (int)chap->sess_key_len, chap->sess_key); 655 return 0; 656 } 657 658 static void nvme_auth_reset_dhchap(struct nvme_dhchap_queue_context *chap) 659 { 660 nvme_auth_free_key(chap->transformed_key); 661 chap->transformed_key = NULL; 662 kfree_sensitive(chap->host_key); 663 chap->host_key = NULL; 664 chap->host_key_len = 0; 665 kfree_sensitive(chap->ctrl_key); 666 chap->ctrl_key = NULL; 667 chap->ctrl_key_len = 0; 668 kfree_sensitive(chap->sess_key); 669 chap->sess_key = NULL; 670 chap->sess_key_len = 0; 671 chap->status = 0; 672 chap->error = 0; 673 chap->s1 = 0; 674 chap->s2 = 0; 675 chap->bi_directional = false; 676 chap->transaction = 0; 677 memset(chap->c1, 0, sizeof(chap->c1)); 678 memset(chap->c2, 0, sizeof(chap->c2)); 679 mempool_free(chap->buf, nvme_chap_buf_pool); 680 chap->buf = NULL; 681 } 682 683 static void nvme_auth_free_dhchap(struct nvme_dhchap_queue_context *chap) 684 { 685 nvme_auth_reset_dhchap(chap); 686 chap->authenticated = false; 687 if (chap->shash_tfm) 688 crypto_free_shash(chap->shash_tfm); 689 if (chap->dh_tfm) 690 crypto_free_kpp(chap->dh_tfm); 691 } 692 693 void nvme_auth_revoke_tls_key(struct nvme_ctrl *ctrl) 694 { 695 dev_dbg(ctrl->device, "Wipe generated TLS PSK %08x\n", 696 key_serial(ctrl->opts->tls_key)); 697 key_revoke(ctrl->opts->tls_key); 698 key_put(ctrl->opts->tls_key); 699 ctrl->opts->tls_key = NULL; 700 } 701 EXPORT_SYMBOL_GPL(nvme_auth_revoke_tls_key); 702 703 static int nvme_auth_secure_concat(struct nvme_ctrl *ctrl, 704 struct nvme_dhchap_queue_context *chap) 705 { 706 u8 *psk, *digest, *tls_psk; 707 struct key *tls_key; 708 size_t psk_len; 709 int ret = 0; 710 711 if (!chap->sess_key) { 712 dev_warn(ctrl->device, 713 "%s: qid %d no session key negotiated\n", 714 __func__, chap->qid); 715 return -ENOKEY; 716 } 717 718 if (chap->qid) { 719 dev_warn(ctrl->device, 720 "qid %d: secure concatenation not supported on I/O queues\n", 721 chap->qid); 722 return -EINVAL; 723 } 724 ret = nvme_auth_generate_psk(chap->hash_id, chap->sess_key, 725 chap->sess_key_len, 726 chap->c1, chap->c2, 727 chap->hash_len, &psk, &psk_len); 728 if (ret) { 729 dev_warn(ctrl->device, 730 "%s: qid %d failed to generate PSK, error %d\n", 731 __func__, chap->qid, ret); 732 return ret; 733 } 734 dev_dbg(ctrl->device, 735 "%s: generated psk %*ph\n", __func__, (int)psk_len, psk); 736 737 ret = nvme_auth_generate_digest(chap->hash_id, psk, psk_len, 738 ctrl->opts->subsysnqn, 739 ctrl->opts->host->nqn, &digest); 740 if (ret) { 741 dev_warn(ctrl->device, 742 "%s: qid %d failed to generate digest, error %d\n", 743 __func__, chap->qid, ret); 744 goto out_free_psk; 745 } 746 dev_dbg(ctrl->device, "%s: generated digest %s\n", 747 __func__, digest); 748 ret = nvme_auth_derive_tls_psk(chap->hash_id, psk, psk_len, 749 digest, &tls_psk); 750 if (ret) { 751 dev_warn(ctrl->device, 752 "%s: qid %d failed to derive TLS psk, error %d\n", 753 __func__, chap->qid, ret); 754 goto out_free_digest; 755 } 756 757 tls_key = nvme_tls_psk_refresh(ctrl->opts->keyring, 758 ctrl->opts->host->nqn, 759 ctrl->opts->subsysnqn, chap->hash_id, 760 tls_psk, psk_len, digest); 761 if (IS_ERR(tls_key)) { 762 ret = PTR_ERR(tls_key); 763 dev_warn(ctrl->device, 764 "%s: qid %d failed to insert generated key, error %d\n", 765 __func__, chap->qid, ret); 766 tls_key = NULL; 767 } 768 kfree_sensitive(tls_psk); 769 if (ctrl->opts->tls_key) 770 nvme_auth_revoke_tls_key(ctrl); 771 ctrl->opts->tls_key = tls_key; 772 out_free_digest: 773 kfree_sensitive(digest); 774 out_free_psk: 775 kfree_sensitive(psk); 776 return ret; 777 } 778 779 static void nvme_queue_auth_work(struct work_struct *work) 780 { 781 struct nvme_dhchap_queue_context *chap = 782 container_of(work, struct nvme_dhchap_queue_context, auth_work); 783 struct nvme_ctrl *ctrl = chap->ctrl; 784 size_t tl; 785 int ret = 0; 786 787 /* 788 * Allocate a large enough buffer for the entire negotiation: 789 * 4k is enough to ffdhe8192. 790 */ 791 chap->buf = mempool_alloc(nvme_chap_buf_pool, GFP_KERNEL); 792 if (!chap->buf) { 793 chap->error = -ENOMEM; 794 return; 795 } 796 797 chap->transaction = ctrl->transaction++; 798 799 /* DH-HMAC-CHAP Step 1: send negotiate */ 800 dev_dbg(ctrl->device, "%s: qid %d send negotiate\n", 801 __func__, chap->qid); 802 ret = nvme_auth_set_dhchap_negotiate_data(ctrl, chap); 803 if (ret < 0) { 804 chap->error = ret; 805 return; 806 } 807 tl = ret; 808 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true); 809 if (ret) { 810 chap->error = ret; 811 return; 812 } 813 814 /* DH-HMAC-CHAP Step 2: receive challenge */ 815 dev_dbg(ctrl->device, "%s: qid %d receive challenge\n", 816 __func__, chap->qid); 817 818 memset(chap->buf, 0, CHAP_BUF_SIZE); 819 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, CHAP_BUF_SIZE, 820 false); 821 if (ret) { 822 dev_warn(ctrl->device, 823 "qid %d failed to receive challenge, %s %d\n", 824 chap->qid, ret < 0 ? "error" : "nvme status", ret); 825 chap->error = ret; 826 return; 827 } 828 ret = nvme_auth_receive_validate(ctrl, chap->qid, chap->buf, chap->transaction, 829 NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE); 830 if (ret) { 831 chap->status = ret; 832 chap->error = -EKEYREJECTED; 833 return; 834 } 835 836 ret = nvme_auth_process_dhchap_challenge(ctrl, chap); 837 if (ret) { 838 /* Invalid challenge parameters */ 839 chap->error = ret; 840 goto fail2; 841 } 842 843 if (chap->ctrl_key_len) { 844 dev_dbg(ctrl->device, 845 "%s: qid %d DH exponential\n", 846 __func__, chap->qid); 847 ret = nvme_auth_dhchap_exponential(ctrl, chap); 848 if (ret) { 849 chap->error = ret; 850 goto fail2; 851 } 852 } 853 854 dev_dbg(ctrl->device, "%s: qid %d host response\n", 855 __func__, chap->qid); 856 mutex_lock(&ctrl->dhchap_auth_mutex); 857 ret = nvme_auth_dhchap_setup_host_response(ctrl, chap); 858 mutex_unlock(&ctrl->dhchap_auth_mutex); 859 if (ret) { 860 chap->error = ret; 861 goto fail2; 862 } 863 864 /* DH-HMAC-CHAP Step 3: send reply */ 865 dev_dbg(ctrl->device, "%s: qid %d send reply\n", 866 __func__, chap->qid); 867 ret = nvme_auth_set_dhchap_reply_data(ctrl, chap); 868 if (ret < 0) { 869 chap->error = ret; 870 goto fail2; 871 } 872 873 tl = ret; 874 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true); 875 if (ret) { 876 chap->error = ret; 877 goto fail2; 878 } 879 880 /* DH-HMAC-CHAP Step 4: receive success1 */ 881 dev_dbg(ctrl->device, "%s: qid %d receive success1\n", 882 __func__, chap->qid); 883 884 memset(chap->buf, 0, CHAP_BUF_SIZE); 885 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, CHAP_BUF_SIZE, 886 false); 887 if (ret) { 888 dev_warn(ctrl->device, 889 "qid %d failed to receive success1, %s %d\n", 890 chap->qid, ret < 0 ? "error" : "nvme status", ret); 891 chap->error = ret; 892 return; 893 } 894 ret = nvme_auth_receive_validate(ctrl, chap->qid, 895 chap->buf, chap->transaction, 896 NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1); 897 if (ret) { 898 chap->status = ret; 899 chap->error = -EKEYREJECTED; 900 return; 901 } 902 903 mutex_lock(&ctrl->dhchap_auth_mutex); 904 if (ctrl->ctrl_key) { 905 dev_dbg(ctrl->device, 906 "%s: qid %d controller response\n", 907 __func__, chap->qid); 908 ret = nvme_auth_dhchap_setup_ctrl_response(ctrl, chap); 909 if (ret) { 910 mutex_unlock(&ctrl->dhchap_auth_mutex); 911 chap->error = ret; 912 goto fail2; 913 } 914 } 915 mutex_unlock(&ctrl->dhchap_auth_mutex); 916 917 ret = nvme_auth_process_dhchap_success1(ctrl, chap); 918 if (ret) { 919 /* Controller authentication failed */ 920 chap->error = -EKEYREJECTED; 921 goto fail2; 922 } 923 924 if (chap->bi_directional) { 925 /* DH-HMAC-CHAP Step 5: send success2 */ 926 dev_dbg(ctrl->device, "%s: qid %d send success2\n", 927 __func__, chap->qid); 928 tl = nvme_auth_set_dhchap_success2_data(ctrl, chap); 929 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true); 930 if (ret) 931 chap->error = ret; 932 } 933 if (!ret) { 934 chap->error = 0; 935 chap->authenticated = true; 936 if (ctrl->opts->concat && 937 (ret = nvme_auth_secure_concat(ctrl, chap))) { 938 dev_warn(ctrl->device, 939 "%s: qid %d failed to enable secure concatenation\n", 940 __func__, chap->qid); 941 chap->error = ret; 942 chap->authenticated = false; 943 } 944 return; 945 } 946 947 fail2: 948 if (chap->status == 0) 949 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 950 dev_dbg(ctrl->device, "%s: qid %d send failure2, status %x\n", 951 __func__, chap->qid, chap->status); 952 tl = nvme_auth_set_dhchap_failure2_data(ctrl, chap); 953 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true); 954 /* 955 * only update error if send failure2 failed and no other 956 * error had been set during authentication. 957 */ 958 if (ret && !chap->error) 959 chap->error = ret; 960 } 961 962 int nvme_auth_negotiate(struct nvme_ctrl *ctrl, int qid) 963 { 964 struct nvme_dhchap_queue_context *chap; 965 966 if (!ctrl->host_key) { 967 dev_warn(ctrl->device, "qid %d: no key\n", qid); 968 return -ENOKEY; 969 } 970 971 if (ctrl->opts->dhchap_ctrl_secret && !ctrl->ctrl_key) { 972 dev_warn(ctrl->device, "qid %d: invalid ctrl key\n", qid); 973 return -ENOKEY; 974 } 975 976 chap = &ctrl->dhchap_ctxs[qid]; 977 cancel_work_sync(&chap->auth_work); 978 queue_work(nvme_auth_wq, &chap->auth_work); 979 return 0; 980 } 981 EXPORT_SYMBOL_GPL(nvme_auth_negotiate); 982 983 int nvme_auth_wait(struct nvme_ctrl *ctrl, int qid) 984 { 985 struct nvme_dhchap_queue_context *chap; 986 int ret; 987 988 chap = &ctrl->dhchap_ctxs[qid]; 989 flush_work(&chap->auth_work); 990 ret = chap->error; 991 /* clear sensitive info */ 992 nvme_auth_reset_dhchap(chap); 993 return ret; 994 } 995 EXPORT_SYMBOL_GPL(nvme_auth_wait); 996 997 static void nvme_ctrl_auth_work(struct work_struct *work) 998 { 999 struct nvme_ctrl *ctrl = 1000 container_of(work, struct nvme_ctrl, dhchap_auth_work); 1001 int ret, q; 1002 1003 /* 1004 * If the ctrl is no connected, bail as reconnect will handle 1005 * authentication. 1006 */ 1007 if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE) 1008 return; 1009 1010 /* Authenticate admin queue first */ 1011 ret = nvme_auth_negotiate(ctrl, 0); 1012 if (ret) { 1013 dev_warn(ctrl->device, 1014 "qid 0: error %d setting up authentication\n", ret); 1015 return; 1016 } 1017 ret = nvme_auth_wait(ctrl, 0); 1018 if (ret) { 1019 dev_warn(ctrl->device, 1020 "qid 0: authentication failed\n"); 1021 return; 1022 } 1023 /* 1024 * Only run authentication on the admin queue for secure concatenation. 1025 */ 1026 if (ctrl->opts->concat) 1027 return; 1028 1029 for (q = 1; q < ctrl->queue_count; q++) { 1030 struct nvme_dhchap_queue_context *chap = 1031 &ctrl->dhchap_ctxs[q]; 1032 /* 1033 * Skip re-authentication if the queue had 1034 * not been authenticated initially. 1035 */ 1036 if (!chap->authenticated) 1037 continue; 1038 cancel_work_sync(&chap->auth_work); 1039 queue_work(nvme_auth_wq, &chap->auth_work); 1040 } 1041 1042 /* 1043 * Failure is a soft-state; credentials remain valid until 1044 * the controller terminates the connection. 1045 */ 1046 for (q = 1; q < ctrl->queue_count; q++) { 1047 struct nvme_dhchap_queue_context *chap = 1048 &ctrl->dhchap_ctxs[q]; 1049 if (!chap->authenticated) 1050 continue; 1051 flush_work(&chap->auth_work); 1052 ret = chap->error; 1053 nvme_auth_reset_dhchap(chap); 1054 if (ret) 1055 dev_warn(ctrl->device, 1056 "qid %d: authentication failed\n", q); 1057 } 1058 } 1059 1060 int nvme_auth_init_ctrl(struct nvme_ctrl *ctrl) 1061 { 1062 struct nvme_dhchap_queue_context *chap; 1063 int i, ret; 1064 1065 mutex_init(&ctrl->dhchap_auth_mutex); 1066 INIT_WORK(&ctrl->dhchap_auth_work, nvme_ctrl_auth_work); 1067 if (!ctrl->opts) 1068 return 0; 1069 ret = nvme_auth_generate_key(ctrl->opts->dhchap_secret, 1070 &ctrl->host_key); 1071 if (ret) 1072 return ret; 1073 ret = nvme_auth_generate_key(ctrl->opts->dhchap_ctrl_secret, 1074 &ctrl->ctrl_key); 1075 if (ret) 1076 goto err_free_dhchap_secret; 1077 1078 if (!ctrl->opts->dhchap_secret && !ctrl->opts->dhchap_ctrl_secret) 1079 return 0; 1080 1081 ctrl->dhchap_ctxs = kvcalloc(ctrl_max_dhchaps(ctrl), 1082 sizeof(*chap), GFP_KERNEL); 1083 if (!ctrl->dhchap_ctxs) { 1084 ret = -ENOMEM; 1085 goto err_free_dhchap_ctrl_secret; 1086 } 1087 1088 for (i = 0; i < ctrl_max_dhchaps(ctrl); i++) { 1089 chap = &ctrl->dhchap_ctxs[i]; 1090 chap->qid = i; 1091 chap->ctrl = ctrl; 1092 chap->authenticated = false; 1093 INIT_WORK(&chap->auth_work, nvme_queue_auth_work); 1094 } 1095 1096 return 0; 1097 err_free_dhchap_ctrl_secret: 1098 nvme_auth_free_key(ctrl->ctrl_key); 1099 ctrl->ctrl_key = NULL; 1100 err_free_dhchap_secret: 1101 nvme_auth_free_key(ctrl->host_key); 1102 ctrl->host_key = NULL; 1103 return ret; 1104 } 1105 EXPORT_SYMBOL_GPL(nvme_auth_init_ctrl); 1106 1107 void nvme_auth_stop(struct nvme_ctrl *ctrl) 1108 { 1109 cancel_work_sync(&ctrl->dhchap_auth_work); 1110 } 1111 EXPORT_SYMBOL_GPL(nvme_auth_stop); 1112 1113 void nvme_auth_free(struct nvme_ctrl *ctrl) 1114 { 1115 int i; 1116 1117 if (ctrl->dhchap_ctxs) { 1118 for (i = 0; i < ctrl_max_dhchaps(ctrl); i++) 1119 nvme_auth_free_dhchap(&ctrl->dhchap_ctxs[i]); 1120 kfree(ctrl->dhchap_ctxs); 1121 } 1122 if (ctrl->host_key) { 1123 nvme_auth_free_key(ctrl->host_key); 1124 ctrl->host_key = NULL; 1125 } 1126 if (ctrl->ctrl_key) { 1127 nvme_auth_free_key(ctrl->ctrl_key); 1128 ctrl->ctrl_key = NULL; 1129 } 1130 } 1131 EXPORT_SYMBOL_GPL(nvme_auth_free); 1132 1133 int __init nvme_init_auth(void) 1134 { 1135 nvme_auth_wq = alloc_workqueue("nvme-auth-wq", 1136 WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0); 1137 if (!nvme_auth_wq) 1138 return -ENOMEM; 1139 1140 nvme_chap_buf_cache = kmem_cache_create("nvme-chap-buf-cache", 1141 CHAP_BUF_SIZE, 0, SLAB_HWCACHE_ALIGN, NULL); 1142 if (!nvme_chap_buf_cache) 1143 goto err_destroy_workqueue; 1144 1145 nvme_chap_buf_pool = mempool_create(16, mempool_alloc_slab, 1146 mempool_free_slab, nvme_chap_buf_cache); 1147 if (!nvme_chap_buf_pool) 1148 goto err_destroy_chap_buf_cache; 1149 1150 return 0; 1151 err_destroy_chap_buf_cache: 1152 kmem_cache_destroy(nvme_chap_buf_cache); 1153 err_destroy_workqueue: 1154 destroy_workqueue(nvme_auth_wq); 1155 return -ENOMEM; 1156 } 1157 1158 void __exit nvme_exit_auth(void) 1159 { 1160 mempool_destroy(nvme_chap_buf_pool); 1161 kmem_cache_destroy(nvme_chap_buf_cache); 1162 destroy_workqueue(nvme_auth_wq); 1163 } 1164