1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2020 Hannes Reinecke, SUSE Linux 4 */ 5 6 #include <linux/crc32.h> 7 #include <linux/base64.h> 8 #include <linux/prandom.h> 9 #include <asm/unaligned.h> 10 #include <crypto/hash.h> 11 #include <crypto/dh.h> 12 #include "nvme.h" 13 #include "fabrics.h" 14 #include <linux/nvme-auth.h> 15 16 #define CHAP_BUF_SIZE 4096 17 static struct kmem_cache *nvme_chap_buf_cache; 18 static mempool_t *nvme_chap_buf_pool; 19 20 struct nvme_dhchap_queue_context { 21 struct list_head entry; 22 struct work_struct auth_work; 23 struct nvme_ctrl *ctrl; 24 struct crypto_shash *shash_tfm; 25 struct crypto_kpp *dh_tfm; 26 struct nvme_dhchap_key *transformed_key; 27 void *buf; 28 int qid; 29 int error; 30 u32 s1; 31 u32 s2; 32 bool bi_directional; 33 u16 transaction; 34 u8 status; 35 u8 dhgroup_id; 36 u8 hash_id; 37 size_t hash_len; 38 u8 c1[64]; 39 u8 c2[64]; 40 u8 response[64]; 41 u8 *ctrl_key; 42 u8 *host_key; 43 u8 *sess_key; 44 int ctrl_key_len; 45 int host_key_len; 46 int sess_key_len; 47 }; 48 49 static struct workqueue_struct *nvme_auth_wq; 50 51 static inline int ctrl_max_dhchaps(struct nvme_ctrl *ctrl) 52 { 53 return ctrl->opts->nr_io_queues + ctrl->opts->nr_write_queues + 54 ctrl->opts->nr_poll_queues + 1; 55 } 56 57 static int nvme_auth_submit(struct nvme_ctrl *ctrl, int qid, 58 void *data, size_t data_len, bool auth_send) 59 { 60 struct nvme_command cmd = {}; 61 nvme_submit_flags_t flags = NVME_SUBMIT_RETRY; 62 struct request_queue *q = ctrl->fabrics_q; 63 int ret; 64 65 if (qid != 0) { 66 flags |= NVME_SUBMIT_NOWAIT | NVME_SUBMIT_RESERVED; 67 q = ctrl->connect_q; 68 } 69 70 cmd.auth_common.opcode = nvme_fabrics_command; 71 cmd.auth_common.secp = NVME_AUTH_DHCHAP_PROTOCOL_IDENTIFIER; 72 cmd.auth_common.spsp0 = 0x01; 73 cmd.auth_common.spsp1 = 0x01; 74 if (auth_send) { 75 cmd.auth_send.fctype = nvme_fabrics_type_auth_send; 76 cmd.auth_send.tl = cpu_to_le32(data_len); 77 } else { 78 cmd.auth_receive.fctype = nvme_fabrics_type_auth_receive; 79 cmd.auth_receive.al = cpu_to_le32(data_len); 80 } 81 82 ret = __nvme_submit_sync_cmd(q, &cmd, NULL, data, data_len, 83 qid == 0 ? NVME_QID_ANY : qid, flags); 84 if (ret > 0) 85 dev_warn(ctrl->device, 86 "qid %d auth_send failed with status %d\n", qid, ret); 87 else if (ret < 0) 88 dev_err(ctrl->device, 89 "qid %d auth_send failed with error %d\n", qid, ret); 90 return ret; 91 } 92 93 static int nvme_auth_receive_validate(struct nvme_ctrl *ctrl, int qid, 94 struct nvmf_auth_dhchap_failure_data *data, 95 u16 transaction, u8 expected_msg) 96 { 97 dev_dbg(ctrl->device, "%s: qid %d auth_type %d auth_id %x\n", 98 __func__, qid, data->auth_type, data->auth_id); 99 100 if (data->auth_type == NVME_AUTH_COMMON_MESSAGES && 101 data->auth_id == NVME_AUTH_DHCHAP_MESSAGE_FAILURE1) { 102 return data->rescode_exp; 103 } 104 if (data->auth_type != NVME_AUTH_DHCHAP_MESSAGES || 105 data->auth_id != expected_msg) { 106 dev_warn(ctrl->device, 107 "qid %d invalid message %02x/%02x\n", 108 qid, data->auth_type, data->auth_id); 109 return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE; 110 } 111 if (le16_to_cpu(data->t_id) != transaction) { 112 dev_warn(ctrl->device, 113 "qid %d invalid transaction ID %d\n", 114 qid, le16_to_cpu(data->t_id)); 115 return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE; 116 } 117 return 0; 118 } 119 120 static int nvme_auth_set_dhchap_negotiate_data(struct nvme_ctrl *ctrl, 121 struct nvme_dhchap_queue_context *chap) 122 { 123 struct nvmf_auth_dhchap_negotiate_data *data = chap->buf; 124 size_t size = sizeof(*data) + sizeof(union nvmf_auth_protocol); 125 126 if (size > CHAP_BUF_SIZE) { 127 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 128 return -EINVAL; 129 } 130 memset((u8 *)chap->buf, 0, size); 131 data->auth_type = NVME_AUTH_COMMON_MESSAGES; 132 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE; 133 data->t_id = cpu_to_le16(chap->transaction); 134 data->sc_c = 0; /* No secure channel concatenation */ 135 data->napd = 1; 136 data->auth_protocol[0].dhchap.authid = NVME_AUTH_DHCHAP_AUTH_ID; 137 data->auth_protocol[0].dhchap.halen = 3; 138 data->auth_protocol[0].dhchap.dhlen = 6; 139 data->auth_protocol[0].dhchap.idlist[0] = NVME_AUTH_HASH_SHA256; 140 data->auth_protocol[0].dhchap.idlist[1] = NVME_AUTH_HASH_SHA384; 141 data->auth_protocol[0].dhchap.idlist[2] = NVME_AUTH_HASH_SHA512; 142 data->auth_protocol[0].dhchap.idlist[30] = NVME_AUTH_DHGROUP_NULL; 143 data->auth_protocol[0].dhchap.idlist[31] = NVME_AUTH_DHGROUP_2048; 144 data->auth_protocol[0].dhchap.idlist[32] = NVME_AUTH_DHGROUP_3072; 145 data->auth_protocol[0].dhchap.idlist[33] = NVME_AUTH_DHGROUP_4096; 146 data->auth_protocol[0].dhchap.idlist[34] = NVME_AUTH_DHGROUP_6144; 147 data->auth_protocol[0].dhchap.idlist[35] = NVME_AUTH_DHGROUP_8192; 148 149 return size; 150 } 151 152 static int nvme_auth_process_dhchap_challenge(struct nvme_ctrl *ctrl, 153 struct nvme_dhchap_queue_context *chap) 154 { 155 struct nvmf_auth_dhchap_challenge_data *data = chap->buf; 156 u16 dhvlen = le16_to_cpu(data->dhvlen); 157 size_t size = sizeof(*data) + data->hl + dhvlen; 158 const char *gid_name = nvme_auth_dhgroup_name(data->dhgid); 159 const char *hmac_name, *kpp_name; 160 161 if (size > CHAP_BUF_SIZE) { 162 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 163 return -EINVAL; 164 } 165 166 hmac_name = nvme_auth_hmac_name(data->hashid); 167 if (!hmac_name) { 168 dev_warn(ctrl->device, 169 "qid %d: invalid HASH ID %d\n", 170 chap->qid, data->hashid); 171 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE; 172 return -EPROTO; 173 } 174 175 if (chap->hash_id == data->hashid && chap->shash_tfm && 176 !strcmp(crypto_shash_alg_name(chap->shash_tfm), hmac_name) && 177 crypto_shash_digestsize(chap->shash_tfm) == data->hl) { 178 dev_dbg(ctrl->device, 179 "qid %d: reuse existing hash %s\n", 180 chap->qid, hmac_name); 181 goto select_kpp; 182 } 183 184 /* Reset if hash cannot be reused */ 185 if (chap->shash_tfm) { 186 crypto_free_shash(chap->shash_tfm); 187 chap->hash_id = 0; 188 chap->hash_len = 0; 189 } 190 chap->shash_tfm = crypto_alloc_shash(hmac_name, 0, 191 CRYPTO_ALG_ALLOCATES_MEMORY); 192 if (IS_ERR(chap->shash_tfm)) { 193 dev_warn(ctrl->device, 194 "qid %d: failed to allocate hash %s, error %ld\n", 195 chap->qid, hmac_name, PTR_ERR(chap->shash_tfm)); 196 chap->shash_tfm = NULL; 197 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 198 return -ENOMEM; 199 } 200 201 if (crypto_shash_digestsize(chap->shash_tfm) != data->hl) { 202 dev_warn(ctrl->device, 203 "qid %d: invalid hash length %d\n", 204 chap->qid, data->hl); 205 crypto_free_shash(chap->shash_tfm); 206 chap->shash_tfm = NULL; 207 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE; 208 return -EPROTO; 209 } 210 211 chap->hash_id = data->hashid; 212 chap->hash_len = data->hl; 213 dev_dbg(ctrl->device, "qid %d: selected hash %s\n", 214 chap->qid, hmac_name); 215 216 select_kpp: 217 kpp_name = nvme_auth_dhgroup_kpp(data->dhgid); 218 if (!kpp_name) { 219 dev_warn(ctrl->device, 220 "qid %d: invalid DH group id %d\n", 221 chap->qid, data->dhgid); 222 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE; 223 /* Leave previous dh_tfm intact */ 224 return -EPROTO; 225 } 226 227 if (chap->dhgroup_id == data->dhgid && 228 (data->dhgid == NVME_AUTH_DHGROUP_NULL || chap->dh_tfm)) { 229 dev_dbg(ctrl->device, 230 "qid %d: reuse existing DH group %s\n", 231 chap->qid, gid_name); 232 goto skip_kpp; 233 } 234 235 /* Reset dh_tfm if it can't be reused */ 236 if (chap->dh_tfm) { 237 crypto_free_kpp(chap->dh_tfm); 238 chap->dh_tfm = NULL; 239 } 240 241 if (data->dhgid != NVME_AUTH_DHGROUP_NULL) { 242 if (dhvlen == 0) { 243 dev_warn(ctrl->device, 244 "qid %d: empty DH value\n", 245 chap->qid); 246 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE; 247 return -EPROTO; 248 } 249 250 chap->dh_tfm = crypto_alloc_kpp(kpp_name, 0, 0); 251 if (IS_ERR(chap->dh_tfm)) { 252 int ret = PTR_ERR(chap->dh_tfm); 253 254 dev_warn(ctrl->device, 255 "qid %d: error %d initializing DH group %s\n", 256 chap->qid, ret, gid_name); 257 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE; 258 chap->dh_tfm = NULL; 259 return ret; 260 } 261 dev_dbg(ctrl->device, "qid %d: selected DH group %s\n", 262 chap->qid, gid_name); 263 } else if (dhvlen != 0) { 264 dev_warn(ctrl->device, 265 "qid %d: invalid DH value for NULL DH\n", 266 chap->qid); 267 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 268 return -EPROTO; 269 } 270 chap->dhgroup_id = data->dhgid; 271 272 skip_kpp: 273 chap->s1 = le32_to_cpu(data->seqnum); 274 memcpy(chap->c1, data->cval, chap->hash_len); 275 if (dhvlen) { 276 chap->ctrl_key = kmalloc(dhvlen, GFP_KERNEL); 277 if (!chap->ctrl_key) { 278 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 279 return -ENOMEM; 280 } 281 chap->ctrl_key_len = dhvlen; 282 memcpy(chap->ctrl_key, data->cval + chap->hash_len, 283 dhvlen); 284 dev_dbg(ctrl->device, "ctrl public key %*ph\n", 285 (int)chap->ctrl_key_len, chap->ctrl_key); 286 } 287 288 return 0; 289 } 290 291 static int nvme_auth_set_dhchap_reply_data(struct nvme_ctrl *ctrl, 292 struct nvme_dhchap_queue_context *chap) 293 { 294 struct nvmf_auth_dhchap_reply_data *data = chap->buf; 295 size_t size = sizeof(*data); 296 297 size += 2 * chap->hash_len; 298 299 if (chap->host_key_len) 300 size += chap->host_key_len; 301 302 if (size > CHAP_BUF_SIZE) { 303 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 304 return -EINVAL; 305 } 306 307 memset(chap->buf, 0, size); 308 data->auth_type = NVME_AUTH_DHCHAP_MESSAGES; 309 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_REPLY; 310 data->t_id = cpu_to_le16(chap->transaction); 311 data->hl = chap->hash_len; 312 data->dhvlen = cpu_to_le16(chap->host_key_len); 313 memcpy(data->rval, chap->response, chap->hash_len); 314 if (ctrl->ctrl_key) { 315 chap->bi_directional = true; 316 get_random_bytes(chap->c2, chap->hash_len); 317 data->cvalid = 1; 318 memcpy(data->rval + chap->hash_len, chap->c2, 319 chap->hash_len); 320 dev_dbg(ctrl->device, "%s: qid %d ctrl challenge %*ph\n", 321 __func__, chap->qid, (int)chap->hash_len, chap->c2); 322 } else { 323 memset(chap->c2, 0, chap->hash_len); 324 } 325 chap->s2 = nvme_auth_get_seqnum(); 326 data->seqnum = cpu_to_le32(chap->s2); 327 if (chap->host_key_len) { 328 dev_dbg(ctrl->device, "%s: qid %d host public key %*ph\n", 329 __func__, chap->qid, 330 chap->host_key_len, chap->host_key); 331 memcpy(data->rval + 2 * chap->hash_len, chap->host_key, 332 chap->host_key_len); 333 } 334 335 return size; 336 } 337 338 static int nvme_auth_process_dhchap_success1(struct nvme_ctrl *ctrl, 339 struct nvme_dhchap_queue_context *chap) 340 { 341 struct nvmf_auth_dhchap_success1_data *data = chap->buf; 342 size_t size = sizeof(*data) + chap->hash_len; 343 344 if (size > CHAP_BUF_SIZE) { 345 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 346 return -EINVAL; 347 } 348 349 if (data->hl != chap->hash_len) { 350 dev_warn(ctrl->device, 351 "qid %d: invalid hash length %u\n", 352 chap->qid, data->hl); 353 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE; 354 return -EPROTO; 355 } 356 357 /* Just print out information for the admin queue */ 358 if (chap->qid == 0) 359 dev_info(ctrl->device, 360 "qid 0: authenticated with hash %s dhgroup %s\n", 361 nvme_auth_hmac_name(chap->hash_id), 362 nvme_auth_dhgroup_name(chap->dhgroup_id)); 363 364 if (!data->rvalid) 365 return 0; 366 367 /* Validate controller response */ 368 if (memcmp(chap->response, data->rval, data->hl)) { 369 dev_dbg(ctrl->device, "%s: qid %d ctrl response %*ph\n", 370 __func__, chap->qid, (int)chap->hash_len, data->rval); 371 dev_dbg(ctrl->device, "%s: qid %d host response %*ph\n", 372 __func__, chap->qid, (int)chap->hash_len, 373 chap->response); 374 dev_warn(ctrl->device, 375 "qid %d: controller authentication failed\n", 376 chap->qid); 377 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 378 return -ECONNREFUSED; 379 } 380 381 /* Just print out information for the admin queue */ 382 if (chap->qid == 0) 383 dev_info(ctrl->device, 384 "qid 0: controller authenticated\n"); 385 return 0; 386 } 387 388 static int nvme_auth_set_dhchap_success2_data(struct nvme_ctrl *ctrl, 389 struct nvme_dhchap_queue_context *chap) 390 { 391 struct nvmf_auth_dhchap_success2_data *data = chap->buf; 392 size_t size = sizeof(*data); 393 394 memset(chap->buf, 0, size); 395 data->auth_type = NVME_AUTH_DHCHAP_MESSAGES; 396 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2; 397 data->t_id = cpu_to_le16(chap->transaction); 398 399 return size; 400 } 401 402 static int nvme_auth_set_dhchap_failure2_data(struct nvme_ctrl *ctrl, 403 struct nvme_dhchap_queue_context *chap) 404 { 405 struct nvmf_auth_dhchap_failure_data *data = chap->buf; 406 size_t size = sizeof(*data); 407 408 memset(chap->buf, 0, size); 409 data->auth_type = NVME_AUTH_COMMON_MESSAGES; 410 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_FAILURE2; 411 data->t_id = cpu_to_le16(chap->transaction); 412 data->rescode = NVME_AUTH_DHCHAP_FAILURE_REASON_FAILED; 413 data->rescode_exp = chap->status; 414 415 return size; 416 } 417 418 static int nvme_auth_dhchap_setup_host_response(struct nvme_ctrl *ctrl, 419 struct nvme_dhchap_queue_context *chap) 420 { 421 SHASH_DESC_ON_STACK(shash, chap->shash_tfm); 422 u8 buf[4], *challenge = chap->c1; 423 int ret; 424 425 dev_dbg(ctrl->device, "%s: qid %d host response seq %u transaction %d\n", 426 __func__, chap->qid, chap->s1, chap->transaction); 427 428 if (!chap->transformed_key) { 429 chap->transformed_key = nvme_auth_transform_key(ctrl->host_key, 430 ctrl->opts->host->nqn); 431 if (IS_ERR(chap->transformed_key)) { 432 ret = PTR_ERR(chap->transformed_key); 433 chap->transformed_key = NULL; 434 return ret; 435 } 436 } else { 437 dev_dbg(ctrl->device, "%s: qid %d re-using host response\n", 438 __func__, chap->qid); 439 } 440 441 ret = crypto_shash_setkey(chap->shash_tfm, 442 chap->transformed_key->key, chap->transformed_key->len); 443 if (ret) { 444 dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n", 445 chap->qid, ret); 446 goto out; 447 } 448 449 if (chap->dh_tfm) { 450 challenge = kmalloc(chap->hash_len, GFP_KERNEL); 451 if (!challenge) { 452 ret = -ENOMEM; 453 goto out; 454 } 455 ret = nvme_auth_augmented_challenge(chap->hash_id, 456 chap->sess_key, 457 chap->sess_key_len, 458 chap->c1, challenge, 459 chap->hash_len); 460 if (ret) 461 goto out; 462 } 463 464 shash->tfm = chap->shash_tfm; 465 ret = crypto_shash_init(shash); 466 if (ret) 467 goto out; 468 ret = crypto_shash_update(shash, challenge, chap->hash_len); 469 if (ret) 470 goto out; 471 put_unaligned_le32(chap->s1, buf); 472 ret = crypto_shash_update(shash, buf, 4); 473 if (ret) 474 goto out; 475 put_unaligned_le16(chap->transaction, buf); 476 ret = crypto_shash_update(shash, buf, 2); 477 if (ret) 478 goto out; 479 memset(buf, 0, sizeof(buf)); 480 ret = crypto_shash_update(shash, buf, 1); 481 if (ret) 482 goto out; 483 ret = crypto_shash_update(shash, "HostHost", 8); 484 if (ret) 485 goto out; 486 ret = crypto_shash_update(shash, ctrl->opts->host->nqn, 487 strlen(ctrl->opts->host->nqn)); 488 if (ret) 489 goto out; 490 ret = crypto_shash_update(shash, buf, 1); 491 if (ret) 492 goto out; 493 ret = crypto_shash_update(shash, ctrl->opts->subsysnqn, 494 strlen(ctrl->opts->subsysnqn)); 495 if (ret) 496 goto out; 497 ret = crypto_shash_final(shash, chap->response); 498 out: 499 if (challenge != chap->c1) 500 kfree(challenge); 501 return ret; 502 } 503 504 static int nvme_auth_dhchap_setup_ctrl_response(struct nvme_ctrl *ctrl, 505 struct nvme_dhchap_queue_context *chap) 506 { 507 SHASH_DESC_ON_STACK(shash, chap->shash_tfm); 508 struct nvme_dhchap_key *transformed_key; 509 u8 buf[4], *challenge = chap->c2; 510 int ret; 511 512 transformed_key = nvme_auth_transform_key(ctrl->ctrl_key, 513 ctrl->opts->subsysnqn); 514 if (IS_ERR(transformed_key)) { 515 ret = PTR_ERR(transformed_key); 516 return ret; 517 } 518 519 ret = crypto_shash_setkey(chap->shash_tfm, 520 transformed_key->key, transformed_key->len); 521 if (ret) { 522 dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n", 523 chap->qid, ret); 524 goto out; 525 } 526 527 if (chap->dh_tfm) { 528 challenge = kmalloc(chap->hash_len, GFP_KERNEL); 529 if (!challenge) { 530 ret = -ENOMEM; 531 goto out; 532 } 533 ret = nvme_auth_augmented_challenge(chap->hash_id, 534 chap->sess_key, 535 chap->sess_key_len, 536 chap->c2, challenge, 537 chap->hash_len); 538 if (ret) 539 goto out; 540 } 541 dev_dbg(ctrl->device, "%s: qid %d ctrl response seq %u transaction %d\n", 542 __func__, chap->qid, chap->s2, chap->transaction); 543 dev_dbg(ctrl->device, "%s: qid %d challenge %*ph\n", 544 __func__, chap->qid, (int)chap->hash_len, challenge); 545 dev_dbg(ctrl->device, "%s: qid %d subsysnqn %s\n", 546 __func__, chap->qid, ctrl->opts->subsysnqn); 547 dev_dbg(ctrl->device, "%s: qid %d hostnqn %s\n", 548 __func__, chap->qid, ctrl->opts->host->nqn); 549 shash->tfm = chap->shash_tfm; 550 ret = crypto_shash_init(shash); 551 if (ret) 552 goto out; 553 ret = crypto_shash_update(shash, challenge, chap->hash_len); 554 if (ret) 555 goto out; 556 put_unaligned_le32(chap->s2, buf); 557 ret = crypto_shash_update(shash, buf, 4); 558 if (ret) 559 goto out; 560 put_unaligned_le16(chap->transaction, buf); 561 ret = crypto_shash_update(shash, buf, 2); 562 if (ret) 563 goto out; 564 memset(buf, 0, 4); 565 ret = crypto_shash_update(shash, buf, 1); 566 if (ret) 567 goto out; 568 ret = crypto_shash_update(shash, "Controller", 10); 569 if (ret) 570 goto out; 571 ret = crypto_shash_update(shash, ctrl->opts->subsysnqn, 572 strlen(ctrl->opts->subsysnqn)); 573 if (ret) 574 goto out; 575 ret = crypto_shash_update(shash, buf, 1); 576 if (ret) 577 goto out; 578 ret = crypto_shash_update(shash, ctrl->opts->host->nqn, 579 strlen(ctrl->opts->host->nqn)); 580 if (ret) 581 goto out; 582 ret = crypto_shash_final(shash, chap->response); 583 out: 584 if (challenge != chap->c2) 585 kfree(challenge); 586 nvme_auth_free_key(transformed_key); 587 return ret; 588 } 589 590 static int nvme_auth_dhchap_exponential(struct nvme_ctrl *ctrl, 591 struct nvme_dhchap_queue_context *chap) 592 { 593 int ret; 594 595 if (chap->host_key && chap->host_key_len) { 596 dev_dbg(ctrl->device, 597 "qid %d: reusing host key\n", chap->qid); 598 goto gen_sesskey; 599 } 600 ret = nvme_auth_gen_privkey(chap->dh_tfm, chap->dhgroup_id); 601 if (ret < 0) { 602 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 603 return ret; 604 } 605 606 chap->host_key_len = crypto_kpp_maxsize(chap->dh_tfm); 607 608 chap->host_key = kzalloc(chap->host_key_len, GFP_KERNEL); 609 if (!chap->host_key) { 610 chap->host_key_len = 0; 611 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 612 return -ENOMEM; 613 } 614 ret = nvme_auth_gen_pubkey(chap->dh_tfm, 615 chap->host_key, chap->host_key_len); 616 if (ret) { 617 dev_dbg(ctrl->device, 618 "failed to generate public key, error %d\n", ret); 619 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 620 return ret; 621 } 622 623 gen_sesskey: 624 chap->sess_key_len = chap->host_key_len; 625 chap->sess_key = kmalloc(chap->sess_key_len, GFP_KERNEL); 626 if (!chap->sess_key) { 627 chap->sess_key_len = 0; 628 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 629 return -ENOMEM; 630 } 631 632 ret = nvme_auth_gen_shared_secret(chap->dh_tfm, 633 chap->ctrl_key, chap->ctrl_key_len, 634 chap->sess_key, chap->sess_key_len); 635 if (ret) { 636 dev_dbg(ctrl->device, 637 "failed to generate shared secret, error %d\n", ret); 638 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 639 return ret; 640 } 641 dev_dbg(ctrl->device, "shared secret %*ph\n", 642 (int)chap->sess_key_len, chap->sess_key); 643 return 0; 644 } 645 646 static void nvme_auth_reset_dhchap(struct nvme_dhchap_queue_context *chap) 647 { 648 nvme_auth_free_key(chap->transformed_key); 649 chap->transformed_key = NULL; 650 kfree_sensitive(chap->host_key); 651 chap->host_key = NULL; 652 chap->host_key_len = 0; 653 kfree_sensitive(chap->ctrl_key); 654 chap->ctrl_key = NULL; 655 chap->ctrl_key_len = 0; 656 kfree_sensitive(chap->sess_key); 657 chap->sess_key = NULL; 658 chap->sess_key_len = 0; 659 chap->status = 0; 660 chap->error = 0; 661 chap->s1 = 0; 662 chap->s2 = 0; 663 chap->bi_directional = false; 664 chap->transaction = 0; 665 memset(chap->c1, 0, sizeof(chap->c1)); 666 memset(chap->c2, 0, sizeof(chap->c2)); 667 mempool_free(chap->buf, nvme_chap_buf_pool); 668 chap->buf = NULL; 669 } 670 671 static void nvme_auth_free_dhchap(struct nvme_dhchap_queue_context *chap) 672 { 673 nvme_auth_reset_dhchap(chap); 674 if (chap->shash_tfm) 675 crypto_free_shash(chap->shash_tfm); 676 if (chap->dh_tfm) 677 crypto_free_kpp(chap->dh_tfm); 678 } 679 680 static void nvme_queue_auth_work(struct work_struct *work) 681 { 682 struct nvme_dhchap_queue_context *chap = 683 container_of(work, struct nvme_dhchap_queue_context, auth_work); 684 struct nvme_ctrl *ctrl = chap->ctrl; 685 size_t tl; 686 int ret = 0; 687 688 /* 689 * Allocate a large enough buffer for the entire negotiation: 690 * 4k is enough to ffdhe8192. 691 */ 692 chap->buf = mempool_alloc(nvme_chap_buf_pool, GFP_KERNEL); 693 if (!chap->buf) { 694 chap->error = -ENOMEM; 695 return; 696 } 697 698 chap->transaction = ctrl->transaction++; 699 700 /* DH-HMAC-CHAP Step 1: send negotiate */ 701 dev_dbg(ctrl->device, "%s: qid %d send negotiate\n", 702 __func__, chap->qid); 703 ret = nvme_auth_set_dhchap_negotiate_data(ctrl, chap); 704 if (ret < 0) { 705 chap->error = ret; 706 return; 707 } 708 tl = ret; 709 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true); 710 if (ret) { 711 chap->error = ret; 712 return; 713 } 714 715 /* DH-HMAC-CHAP Step 2: receive challenge */ 716 dev_dbg(ctrl->device, "%s: qid %d receive challenge\n", 717 __func__, chap->qid); 718 719 memset(chap->buf, 0, CHAP_BUF_SIZE); 720 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, CHAP_BUF_SIZE, 721 false); 722 if (ret) { 723 dev_warn(ctrl->device, 724 "qid %d failed to receive challenge, %s %d\n", 725 chap->qid, ret < 0 ? "error" : "nvme status", ret); 726 chap->error = ret; 727 return; 728 } 729 ret = nvme_auth_receive_validate(ctrl, chap->qid, chap->buf, chap->transaction, 730 NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE); 731 if (ret) { 732 chap->status = ret; 733 chap->error = -EKEYREJECTED; 734 return; 735 } 736 737 ret = nvme_auth_process_dhchap_challenge(ctrl, chap); 738 if (ret) { 739 /* Invalid challenge parameters */ 740 chap->error = ret; 741 goto fail2; 742 } 743 744 if (chap->ctrl_key_len) { 745 dev_dbg(ctrl->device, 746 "%s: qid %d DH exponential\n", 747 __func__, chap->qid); 748 ret = nvme_auth_dhchap_exponential(ctrl, chap); 749 if (ret) { 750 chap->error = ret; 751 goto fail2; 752 } 753 } 754 755 dev_dbg(ctrl->device, "%s: qid %d host response\n", 756 __func__, chap->qid); 757 mutex_lock(&ctrl->dhchap_auth_mutex); 758 ret = nvme_auth_dhchap_setup_host_response(ctrl, chap); 759 mutex_unlock(&ctrl->dhchap_auth_mutex); 760 if (ret) { 761 chap->error = ret; 762 goto fail2; 763 } 764 765 /* DH-HMAC-CHAP Step 3: send reply */ 766 dev_dbg(ctrl->device, "%s: qid %d send reply\n", 767 __func__, chap->qid); 768 ret = nvme_auth_set_dhchap_reply_data(ctrl, chap); 769 if (ret < 0) { 770 chap->error = ret; 771 goto fail2; 772 } 773 774 tl = ret; 775 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true); 776 if (ret) { 777 chap->error = ret; 778 goto fail2; 779 } 780 781 /* DH-HMAC-CHAP Step 4: receive success1 */ 782 dev_dbg(ctrl->device, "%s: qid %d receive success1\n", 783 __func__, chap->qid); 784 785 memset(chap->buf, 0, CHAP_BUF_SIZE); 786 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, CHAP_BUF_SIZE, 787 false); 788 if (ret) { 789 dev_warn(ctrl->device, 790 "qid %d failed to receive success1, %s %d\n", 791 chap->qid, ret < 0 ? "error" : "nvme status", ret); 792 chap->error = ret; 793 return; 794 } 795 ret = nvme_auth_receive_validate(ctrl, chap->qid, 796 chap->buf, chap->transaction, 797 NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1); 798 if (ret) { 799 chap->status = ret; 800 chap->error = -EKEYREJECTED; 801 return; 802 } 803 804 mutex_lock(&ctrl->dhchap_auth_mutex); 805 if (ctrl->ctrl_key) { 806 dev_dbg(ctrl->device, 807 "%s: qid %d controller response\n", 808 __func__, chap->qid); 809 ret = nvme_auth_dhchap_setup_ctrl_response(ctrl, chap); 810 if (ret) { 811 mutex_unlock(&ctrl->dhchap_auth_mutex); 812 chap->error = ret; 813 goto fail2; 814 } 815 } 816 mutex_unlock(&ctrl->dhchap_auth_mutex); 817 818 ret = nvme_auth_process_dhchap_success1(ctrl, chap); 819 if (ret) { 820 /* Controller authentication failed */ 821 chap->error = -EKEYREJECTED; 822 goto fail2; 823 } 824 825 if (chap->bi_directional) { 826 /* DH-HMAC-CHAP Step 5: send success2 */ 827 dev_dbg(ctrl->device, "%s: qid %d send success2\n", 828 __func__, chap->qid); 829 tl = nvme_auth_set_dhchap_success2_data(ctrl, chap); 830 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true); 831 if (ret) 832 chap->error = ret; 833 } 834 if (!ret) { 835 chap->error = 0; 836 return; 837 } 838 839 fail2: 840 if (chap->status == 0) 841 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 842 dev_dbg(ctrl->device, "%s: qid %d send failure2, status %x\n", 843 __func__, chap->qid, chap->status); 844 tl = nvme_auth_set_dhchap_failure2_data(ctrl, chap); 845 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true); 846 /* 847 * only update error if send failure2 failed and no other 848 * error had been set during authentication. 849 */ 850 if (ret && !chap->error) 851 chap->error = ret; 852 } 853 854 int nvme_auth_negotiate(struct nvme_ctrl *ctrl, int qid) 855 { 856 struct nvme_dhchap_queue_context *chap; 857 858 if (!ctrl->host_key) { 859 dev_warn(ctrl->device, "qid %d: no key\n", qid); 860 return -ENOKEY; 861 } 862 863 if (ctrl->opts->dhchap_ctrl_secret && !ctrl->ctrl_key) { 864 dev_warn(ctrl->device, "qid %d: invalid ctrl key\n", qid); 865 return -ENOKEY; 866 } 867 868 chap = &ctrl->dhchap_ctxs[qid]; 869 cancel_work_sync(&chap->auth_work); 870 queue_work(nvme_auth_wq, &chap->auth_work); 871 return 0; 872 } 873 EXPORT_SYMBOL_GPL(nvme_auth_negotiate); 874 875 int nvme_auth_wait(struct nvme_ctrl *ctrl, int qid) 876 { 877 struct nvme_dhchap_queue_context *chap; 878 int ret; 879 880 chap = &ctrl->dhchap_ctxs[qid]; 881 flush_work(&chap->auth_work); 882 ret = chap->error; 883 /* clear sensitive info */ 884 nvme_auth_reset_dhchap(chap); 885 return ret; 886 } 887 EXPORT_SYMBOL_GPL(nvme_auth_wait); 888 889 static void nvme_ctrl_auth_work(struct work_struct *work) 890 { 891 struct nvme_ctrl *ctrl = 892 container_of(work, struct nvme_ctrl, dhchap_auth_work); 893 int ret, q; 894 895 /* 896 * If the ctrl is no connected, bail as reconnect will handle 897 * authentication. 898 */ 899 if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE) 900 return; 901 902 /* Authenticate admin queue first */ 903 ret = nvme_auth_negotiate(ctrl, 0); 904 if (ret) { 905 dev_warn(ctrl->device, 906 "qid 0: error %d setting up authentication\n", ret); 907 return; 908 } 909 ret = nvme_auth_wait(ctrl, 0); 910 if (ret) { 911 dev_warn(ctrl->device, 912 "qid 0: authentication failed\n"); 913 return; 914 } 915 916 for (q = 1; q < ctrl->queue_count; q++) { 917 ret = nvme_auth_negotiate(ctrl, q); 918 if (ret) { 919 dev_warn(ctrl->device, 920 "qid %d: error %d setting up authentication\n", 921 q, ret); 922 break; 923 } 924 } 925 926 /* 927 * Failure is a soft-state; credentials remain valid until 928 * the controller terminates the connection. 929 */ 930 for (q = 1; q < ctrl->queue_count; q++) { 931 ret = nvme_auth_wait(ctrl, q); 932 if (ret) 933 dev_warn(ctrl->device, 934 "qid %d: authentication failed\n", q); 935 } 936 } 937 938 int nvme_auth_init_ctrl(struct nvme_ctrl *ctrl) 939 { 940 struct nvme_dhchap_queue_context *chap; 941 int i, ret; 942 943 mutex_init(&ctrl->dhchap_auth_mutex); 944 INIT_WORK(&ctrl->dhchap_auth_work, nvme_ctrl_auth_work); 945 if (!ctrl->opts) 946 return 0; 947 ret = nvme_auth_generate_key(ctrl->opts->dhchap_secret, 948 &ctrl->host_key); 949 if (ret) 950 return ret; 951 ret = nvme_auth_generate_key(ctrl->opts->dhchap_ctrl_secret, 952 &ctrl->ctrl_key); 953 if (ret) 954 goto err_free_dhchap_secret; 955 956 if (!ctrl->opts->dhchap_secret && !ctrl->opts->dhchap_ctrl_secret) 957 return 0; 958 959 ctrl->dhchap_ctxs = kvcalloc(ctrl_max_dhchaps(ctrl), 960 sizeof(*chap), GFP_KERNEL); 961 if (!ctrl->dhchap_ctxs) { 962 ret = -ENOMEM; 963 goto err_free_dhchap_ctrl_secret; 964 } 965 966 for (i = 0; i < ctrl_max_dhchaps(ctrl); i++) { 967 chap = &ctrl->dhchap_ctxs[i]; 968 chap->qid = i; 969 chap->ctrl = ctrl; 970 INIT_WORK(&chap->auth_work, nvme_queue_auth_work); 971 } 972 973 return 0; 974 err_free_dhchap_ctrl_secret: 975 nvme_auth_free_key(ctrl->ctrl_key); 976 ctrl->ctrl_key = NULL; 977 err_free_dhchap_secret: 978 nvme_auth_free_key(ctrl->host_key); 979 ctrl->host_key = NULL; 980 return ret; 981 } 982 EXPORT_SYMBOL_GPL(nvme_auth_init_ctrl); 983 984 void nvme_auth_stop(struct nvme_ctrl *ctrl) 985 { 986 cancel_work_sync(&ctrl->dhchap_auth_work); 987 } 988 EXPORT_SYMBOL_GPL(nvme_auth_stop); 989 990 void nvme_auth_free(struct nvme_ctrl *ctrl) 991 { 992 int i; 993 994 if (ctrl->dhchap_ctxs) { 995 for (i = 0; i < ctrl_max_dhchaps(ctrl); i++) 996 nvme_auth_free_dhchap(&ctrl->dhchap_ctxs[i]); 997 kfree(ctrl->dhchap_ctxs); 998 } 999 if (ctrl->host_key) { 1000 nvme_auth_free_key(ctrl->host_key); 1001 ctrl->host_key = NULL; 1002 } 1003 if (ctrl->ctrl_key) { 1004 nvme_auth_free_key(ctrl->ctrl_key); 1005 ctrl->ctrl_key = NULL; 1006 } 1007 } 1008 EXPORT_SYMBOL_GPL(nvme_auth_free); 1009 1010 int __init nvme_init_auth(void) 1011 { 1012 nvme_auth_wq = alloc_workqueue("nvme-auth-wq", 1013 WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0); 1014 if (!nvme_auth_wq) 1015 return -ENOMEM; 1016 1017 nvme_chap_buf_cache = kmem_cache_create("nvme-chap-buf-cache", 1018 CHAP_BUF_SIZE, 0, SLAB_HWCACHE_ALIGN, NULL); 1019 if (!nvme_chap_buf_cache) 1020 goto err_destroy_workqueue; 1021 1022 nvme_chap_buf_pool = mempool_create(16, mempool_alloc_slab, 1023 mempool_free_slab, nvme_chap_buf_cache); 1024 if (!nvme_chap_buf_pool) 1025 goto err_destroy_chap_buf_cache; 1026 1027 return 0; 1028 err_destroy_chap_buf_cache: 1029 kmem_cache_destroy(nvme_chap_buf_cache); 1030 err_destroy_workqueue: 1031 destroy_workqueue(nvme_auth_wq); 1032 return -ENOMEM; 1033 } 1034 1035 void __exit nvme_exit_auth(void) 1036 { 1037 mempool_destroy(nvme_chap_buf_pool); 1038 kmem_cache_destroy(nvme_chap_buf_cache); 1039 destroy_workqueue(nvme_auth_wq); 1040 } 1041