24 	struct nvme_ctrl *ctrl;  member
54 static inline int ctrl_max_dhchaps(struct nvme_ctrl *ctrl)  in ctrl_max_dhchaps()  argument
56 	return ctrl->opts->nr_io_queues + ctrl->opts->nr_write_queues +  in ctrl_max_dhchaps()
57 			ctrl->opts->nr_poll_queues + 1;  in ctrl_max_dhchaps()
60 static int nvme_auth_submit(struct nvme_ctrl *ctrl, int qid,  in nvme_auth_submit()  argument
65 	struct request_queue *q = ctrl->fabrics_q;  in nvme_auth_submit()
70 		q = ctrl->connect_q;  in nvme_auth_submit()
88 		dev_warn(ctrl->device,  in nvme_auth_submit()
91 		dev_err(ctrl->device,  in nvme_auth_submit()
96 static int nvme_auth_receive_validate(struct nvme_ctrl *ctrl, int qid,  in nvme_auth_receive_validate()  argument
100 	dev_dbg(ctrl->device, "%s: qid %d auth_type %d auth_id %x\n",  in nvme_auth_receive_validate()
109 		dev_warn(ctrl->device,  in nvme_auth_receive_validate()
115 		dev_warn(ctrl->device,  in nvme_auth_receive_validate()
123 static int nvme_auth_set_dhchap_negotiate_data(struct nvme_ctrl *ctrl,  in nvme_auth_set_dhchap_negotiate_data()  argument
137 	if (ctrl->opts->concat && chap->qid == 0) {  in nvme_auth_set_dhchap_negotiate_data()
138 		if (ctrl->opts->tls_key)  in nvme_auth_set_dhchap_negotiate_data()
163 static int nvme_auth_process_dhchap_challenge(struct nvme_ctrl *ctrl,  in nvme_auth_process_dhchap_challenge()  argument
179 		dev_warn(ctrl->device,  in nvme_auth_process_dhchap_challenge()
189 		dev_dbg(ctrl->device,  in nvme_auth_process_dhchap_challenge()
204 		dev_warn(ctrl->device,  in nvme_auth_process_dhchap_challenge()
213 		dev_warn(ctrl->device,  in nvme_auth_process_dhchap_challenge()
224 	dev_dbg(ctrl->device, "qid %d: selected hash %s\n",  in nvme_auth_process_dhchap_challenge()
230 		dev_warn(ctrl->device,  in nvme_auth_process_dhchap_challenge()
240 		dev_dbg(ctrl->device,  in nvme_auth_process_dhchap_challenge()
254 			dev_warn(ctrl->device,  in nvme_auth_process_dhchap_challenge()
265 			dev_warn(ctrl->device,  in nvme_auth_process_dhchap_challenge()
272 		dev_dbg(ctrl->device, "qid %d: selected DH group %s\n",  in nvme_auth_process_dhchap_challenge()
275 		dev_warn(ctrl->device,  in nvme_auth_process_dhchap_challenge()
295 		dev_dbg(ctrl->device, "ctrl public key %*ph\n",  in nvme_auth_process_dhchap_challenge()
302 static int nvme_auth_set_dhchap_reply_data(struct nvme_ctrl *ctrl,  in nvme_auth_set_dhchap_reply_data()  argument
325 	if (ctrl->ctrl_key)  in nvme_auth_set_dhchap_reply_data()
327 	if (ctrl->ctrl_key || ctrl->opts->concat) {  in nvme_auth_set_dhchap_reply_data()
332 		dev_dbg(ctrl->device, "%s: qid %d ctrl challenge %*ph\n",  in nvme_auth_set_dhchap_reply_data()
337 	if (ctrl->opts->concat) {  in nvme_auth_set_dhchap_reply_data()
344 		dev_dbg(ctrl->device, "%s: qid %d host public key %*ph\n",  in nvme_auth_set_dhchap_reply_data()
354 static int nvme_auth_process_dhchap_success1(struct nvme_ctrl *ctrl,  in nvme_auth_process_dhchap_success1()  argument
366 		dev_warn(ctrl->device,  in nvme_auth_process_dhchap_success1()
375 		dev_info(ctrl->device,  in nvme_auth_process_dhchap_success1()
385 		dev_dbg(ctrl->device, "%s: qid %d ctrl response %*ph\n",  in nvme_auth_process_dhchap_success1()
387 		dev_dbg(ctrl->device, "%s: qid %d host response %*ph\n",  in nvme_auth_process_dhchap_success1()
390 		dev_warn(ctrl->device,  in nvme_auth_process_dhchap_success1()
399 		dev_info(ctrl->device,  in nvme_auth_process_dhchap_success1()
404 static int nvme_auth_set_dhchap_success2_data(struct nvme_ctrl *ctrl,  in nvme_auth_set_dhchap_success2_data()  argument
418 static int nvme_auth_set_dhchap_failure2_data(struct nvme_ctrl *ctrl,  in nvme_auth_set_dhchap_failure2_data()  argument
434 static int nvme_auth_dhchap_setup_host_response(struct nvme_ctrl *ctrl,  in nvme_auth_dhchap_setup_host_response()  argument
441 	dev_dbg(ctrl->device, "%s: qid %d host response seq %u transaction %d\n",  in nvme_auth_dhchap_setup_host_response()
445 		chap->transformed_key = nvme_auth_transform_key(ctrl->host_key,  in nvme_auth_dhchap_setup_host_response()
446 						ctrl->opts->host->nqn);  in nvme_auth_dhchap_setup_host_response()
453 		dev_dbg(ctrl->device, "%s: qid %d re-using host response\n",  in nvme_auth_dhchap_setup_host_response()
460 		dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n",  in nvme_auth_dhchap_setup_host_response()
502 	ret = crypto_shash_update(shash, ctrl->opts->host->nqn,  in nvme_auth_dhchap_setup_host_response()
503 				  strlen(ctrl->opts->host->nqn));  in nvme_auth_dhchap_setup_host_response()
510 	ret = crypto_shash_update(shash, ctrl->opts->subsysnqn,  in nvme_auth_dhchap_setup_host_response()
511 			    strlen(ctrl->opts->subsysnqn));  in nvme_auth_dhchap_setup_host_response()
521 static int nvme_auth_dhchap_setup_ctrl_response(struct nvme_ctrl *ctrl,  in nvme_auth_dhchap_setup_ctrl_response()  argument
529 	transformed_key = nvme_auth_transform_key(ctrl->ctrl_key,  in nvme_auth_dhchap_setup_ctrl_response()
530 				ctrl->opts->subsysnqn);  in nvme_auth_dhchap_setup_ctrl_response()
539 		dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n",  in nvme_auth_dhchap_setup_ctrl_response()
558 	dev_dbg(ctrl->device, "%s: qid %d ctrl response seq %u transaction %d\n",  in nvme_auth_dhchap_setup_ctrl_response()
560 	dev_dbg(ctrl->device, "%s: qid %d challenge %*ph\n",  in nvme_auth_dhchap_setup_ctrl_response()
562 	dev_dbg(ctrl->device, "%s: qid %d subsysnqn %s\n",  in nvme_auth_dhchap_setup_ctrl_response()
563 		__func__, chap->qid, ctrl->opts->subsysnqn);  in nvme_auth_dhchap_setup_ctrl_response()
564 	dev_dbg(ctrl->device, "%s: qid %d hostnqn %s\n",  in nvme_auth_dhchap_setup_ctrl_response()
565 		__func__, chap->qid, ctrl->opts->host->nqn);  in nvme_auth_dhchap_setup_ctrl_response()
588 	ret = crypto_shash_update(shash, ctrl->opts->subsysnqn,  in nvme_auth_dhchap_setup_ctrl_response()
589 				  strlen(ctrl->opts->subsysnqn));  in nvme_auth_dhchap_setup_ctrl_response()
595 	ret = crypto_shash_update(shash, ctrl->opts->host->nqn,  in nvme_auth_dhchap_setup_ctrl_response()
596 				  strlen(ctrl->opts->host->nqn));  in nvme_auth_dhchap_setup_ctrl_response()
607 static int nvme_auth_dhchap_exponential(struct nvme_ctrl *ctrl,  in nvme_auth_dhchap_exponential()  argument
613 		dev_dbg(ctrl->device,  in nvme_auth_dhchap_exponential()
634 		dev_dbg(ctrl->device,  in nvme_auth_dhchap_exponential()
653 		dev_dbg(ctrl->device,  in nvme_auth_dhchap_exponential()
658 	dev_dbg(ctrl->device, "shared secret %*ph\n",  in nvme_auth_dhchap_exponential()
698 void nvme_auth_revoke_tls_key(struct nvme_ctrl *ctrl)  in nvme_auth_revoke_tls_key()  argument
700 	dev_dbg(ctrl->device, "Wipe generated TLS PSK %08x\n",  in nvme_auth_revoke_tls_key()
701 		key_serial(ctrl->opts->tls_key));  in nvme_auth_revoke_tls_key()
702 	key_revoke(ctrl->opts->tls_key);  in nvme_auth_revoke_tls_key()
703 	key_put(ctrl->opts->tls_key);  in nvme_auth_revoke_tls_key()
704 	ctrl->opts->tls_key = NULL;  in nvme_auth_revoke_tls_key()
708 static int nvme_auth_secure_concat(struct nvme_ctrl *ctrl,  in nvme_auth_secure_concat()  argument
717 		dev_warn(ctrl->device,  in nvme_auth_secure_concat()
724 		dev_warn(ctrl->device,  in nvme_auth_secure_concat()
734 		dev_warn(ctrl->device,  in nvme_auth_secure_concat()
739 	dev_dbg(ctrl->device,  in nvme_auth_secure_concat()
743 					ctrl->opts->subsysnqn,  in nvme_auth_secure_concat()
744 					ctrl->opts->host->nqn, &digest);  in nvme_auth_secure_concat()
746 		dev_warn(ctrl->device,  in nvme_auth_secure_concat()
751 	dev_dbg(ctrl->device, "%s: generated digest %s\n",  in nvme_auth_secure_concat()
756 		dev_warn(ctrl->device,  in nvme_auth_secure_concat()
762 	tls_key = nvme_tls_psk_refresh(ctrl->opts->keyring,  in nvme_auth_secure_concat()
763 				       ctrl->opts->host->nqn,  in nvme_auth_secure_concat()
764 				       ctrl->opts->subsysnqn, chap->hash_id,  in nvme_auth_secure_concat()
768 		dev_warn(ctrl->device,  in nvme_auth_secure_concat()
774 	if (ctrl->opts->tls_key)  in nvme_auth_secure_concat()
775 		nvme_auth_revoke_tls_key(ctrl);  in nvme_auth_secure_concat()
776 	ctrl->opts->tls_key = tls_key;  in nvme_auth_secure_concat()
788 	struct nvme_ctrl *ctrl = chap->ctrl;  in nvme_queue_auth_work()  local
802 	chap->transaction = ctrl->transaction++;  in nvme_queue_auth_work()
805 	dev_dbg(ctrl->device, "%s: qid %d send negotiate\n",  in nvme_queue_auth_work()
807 	ret = nvme_auth_set_dhchap_negotiate_data(ctrl, chap);  in nvme_queue_auth_work()
813 	ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);  in nvme_queue_auth_work()
820 	dev_dbg(ctrl->device, "%s: qid %d receive challenge\n",  in nvme_queue_auth_work()
824 	ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, CHAP_BUF_SIZE,  in nvme_queue_auth_work()
827 		dev_warn(ctrl->device,  in nvme_queue_auth_work()
833 	ret = nvme_auth_receive_validate(ctrl, chap->qid, chap->buf, chap->transaction,  in nvme_queue_auth_work()
841 	ret = nvme_auth_process_dhchap_challenge(ctrl, chap);  in nvme_queue_auth_work()
849 		dev_dbg(ctrl->device,  in nvme_queue_auth_work()
852 		ret = nvme_auth_dhchap_exponential(ctrl, chap);  in nvme_queue_auth_work()
859 	dev_dbg(ctrl->device, "%s: qid %d host response\n",  in nvme_queue_auth_work()
861 	mutex_lock(&ctrl->dhchap_auth_mutex);  in nvme_queue_auth_work()
862 	ret = nvme_auth_dhchap_setup_host_response(ctrl, chap);  in nvme_queue_auth_work()
863 	mutex_unlock(&ctrl->dhchap_auth_mutex);  in nvme_queue_auth_work()
870 	dev_dbg(ctrl->device, "%s: qid %d send reply\n",  in nvme_queue_auth_work()
872 	ret = nvme_auth_set_dhchap_reply_data(ctrl, chap);  in nvme_queue_auth_work()
879 	ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);  in nvme_queue_auth_work()
886 	dev_dbg(ctrl->device, "%s: qid %d receive success1\n",  in nvme_queue_auth_work()
890 	ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, CHAP_BUF_SIZE,  in nvme_queue_auth_work()
893 		dev_warn(ctrl->device,  in nvme_queue_auth_work()
899 	ret = nvme_auth_receive_validate(ctrl, chap->qid,  in nvme_queue_auth_work()
908 	mutex_lock(&ctrl->dhchap_auth_mutex);  in nvme_queue_auth_work()
909 	if (ctrl->ctrl_key) {  in nvme_queue_auth_work()
910 		dev_dbg(ctrl->device,  in nvme_queue_auth_work()
913 		ret = nvme_auth_dhchap_setup_ctrl_response(ctrl, chap);  in nvme_queue_auth_work()
915 			mutex_unlock(&ctrl->dhchap_auth_mutex);  in nvme_queue_auth_work()
920 	mutex_unlock(&ctrl->dhchap_auth_mutex);  in nvme_queue_auth_work()
922 	ret = nvme_auth_process_dhchap_success1(ctrl, chap);  in nvme_queue_auth_work()
931 		dev_dbg(ctrl->device, "%s: qid %d send success2\n",  in nvme_queue_auth_work()
933 		tl = nvme_auth_set_dhchap_success2_data(ctrl, chap);  in nvme_queue_auth_work()
934 		ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);  in nvme_queue_auth_work()
941 		if (ctrl->opts->concat &&  in nvme_queue_auth_work()
942 		    (ret = nvme_auth_secure_concat(ctrl, chap))) {  in nvme_queue_auth_work()
943 			dev_warn(ctrl->device,  in nvme_queue_auth_work()
955 	dev_dbg(ctrl->device, "%s: qid %d send failure2, status %x\n",  in nvme_queue_auth_work()
957 	tl = nvme_auth_set_dhchap_failure2_data(ctrl, chap);  in nvme_queue_auth_work()
958 	ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);  in nvme_queue_auth_work()
967 int nvme_auth_negotiate(struct nvme_ctrl *ctrl, int qid)  in nvme_auth_negotiate()  argument
971 	if (!ctrl->host_key) {  in nvme_auth_negotiate()
972 		dev_warn(ctrl->device, "qid %d: no key\n", qid);  in nvme_auth_negotiate()
976 	if (ctrl->opts->dhchap_ctrl_secret && !ctrl->ctrl_key) {  in nvme_auth_negotiate()
977 		dev_warn(ctrl->device, "qid %d: invalid ctrl key\n", qid);  in nvme_auth_negotiate()
981 	chap = &ctrl->dhchap_ctxs[qid];  in nvme_auth_negotiate()
988 int nvme_auth_wait(struct nvme_ctrl *ctrl, int qid)  in nvme_auth_wait()  argument
993 	chap = &ctrl->dhchap_ctxs[qid];  in nvme_auth_wait()
1004 	struct nvme_ctrl *ctrl =  in nvme_ctrl_auth_work()  local
1009 	 * If the ctrl is no connected, bail as reconnect will handle  in nvme_ctrl_auth_work()
1012 	if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE)  in nvme_ctrl_auth_work()
1016 	ret = nvme_auth_negotiate(ctrl, 0);  in nvme_ctrl_auth_work()
1018 		dev_warn(ctrl->device,  in nvme_ctrl_auth_work()
1022 	ret = nvme_auth_wait(ctrl, 0);  in nvme_ctrl_auth_work()
1024 		dev_warn(ctrl->device,  in nvme_ctrl_auth_work()
1031 	if (ctrl->opts->concat)  in nvme_ctrl_auth_work()
1034 	for (q = 1; q < ctrl->queue_count; q++) {  in nvme_ctrl_auth_work()
1036 			&ctrl->dhchap_ctxs[q];  in nvme_ctrl_auth_work()
1051 	for (q = 1; q < ctrl->queue_count; q++) {  in nvme_ctrl_auth_work()
1053 			&ctrl->dhchap_ctxs[q];  in nvme_ctrl_auth_work()
1060 			dev_warn(ctrl->device,  in nvme_ctrl_auth_work()
1065 int nvme_auth_init_ctrl(struct nvme_ctrl *ctrl)  in nvme_auth_init_ctrl()  argument
1070 	mutex_init(&ctrl->dhchap_auth_mutex);  in nvme_auth_init_ctrl()
1071 	INIT_WORK(&ctrl->dhchap_auth_work, nvme_ctrl_auth_work);  in nvme_auth_init_ctrl()
1072 	if (!ctrl->opts)  in nvme_auth_init_ctrl()
1074 	ret = nvme_auth_generate_key(ctrl->opts->dhchap_secret,  in nvme_auth_init_ctrl()
1075 			&ctrl->host_key);  in nvme_auth_init_ctrl()
1078 	ret = nvme_auth_generate_key(ctrl->opts->dhchap_ctrl_secret,  in nvme_auth_init_ctrl()
1079 			&ctrl->ctrl_key);  in nvme_auth_init_ctrl()
1083 	if (!ctrl->opts->dhchap_secret && !ctrl->opts->dhchap_ctrl_secret)  in nvme_auth_init_ctrl()
1086 	ctrl->dhchap_ctxs = kvcalloc(ctrl_max_dhchaps(ctrl),  in nvme_auth_init_ctrl()
1088 	if (!ctrl->dhchap_ctxs) {  in nvme_auth_init_ctrl()
1093 	for (i = 0; i < ctrl_max_dhchaps(ctrl); i++) {  in nvme_auth_init_ctrl()
1094 		chap = &ctrl->dhchap_ctxs[i];  in nvme_auth_init_ctrl()
1096 		chap->ctrl = ctrl;  in nvme_auth_init_ctrl()
1103 	nvme_auth_free_key(ctrl->ctrl_key);  in nvme_auth_init_ctrl()
1104 	ctrl->ctrl_key = NULL;  in nvme_auth_init_ctrl()
1106 	nvme_auth_free_key(ctrl->host_key);  in nvme_auth_init_ctrl()
1107 	ctrl->host_key = NULL;  in nvme_auth_init_ctrl()
1112 void nvme_auth_stop(struct nvme_ctrl *ctrl)  in nvme_auth_stop()  argument
1114 	cancel_work_sync(&ctrl->dhchap_auth_work);  in nvme_auth_stop()
1118 void nvme_auth_free(struct nvme_ctrl *ctrl)  in nvme_auth_free()  argument
1122 	if (ctrl->dhchap_ctxs) {  in nvme_auth_free()
1123 		for (i = 0; i < ctrl_max_dhchaps(ctrl); i++)  in nvme_auth_free()
1124 			nvme_auth_free_dhchap(&ctrl->dhchap_ctxs[i]);  in nvme_auth_free()
1125 		kfree(ctrl->dhchap_ctxs);  in nvme_auth_free()
1127 	if (ctrl->host_key) {  in nvme_auth_free()
1128 		nvme_auth_free_key(ctrl->host_key);  in nvme_auth_free()
1129 		ctrl->host_key = NULL;  in nvme_auth_free()
1131 	if (ctrl->ctrl_key) {  in nvme_auth_free()
1132 		nvme_auth_free_key(ctrl->ctrl_key);  in nvme_auth_free()
1133 		ctrl->ctrl_key = NULL;  in nvme_auth_free()