106 	struct nvme_tcp_queue	*queue;  member
207 static int nvme_tcp_try_send(struct nvme_tcp_queue *queue);
214 static inline int nvme_tcp_queue_id(struct nvme_tcp_queue *queue)  in nvme_tcp_queue_id()  argument
216 	return queue - queue->ctrl->queues;  in nvme_tcp_queue_id()
235 static inline bool nvme_tcp_queue_tls(struct nvme_tcp_queue *queue)  in nvme_tcp_queue_tls()  argument
240 	return queue->tls_enabled;  in nvme_tcp_queue_tls()
254 static inline struct blk_mq_tags *nvme_tcp_tagset(struct nvme_tcp_queue *queue)  in nvme_tcp_tagset()  argument
256 	u32 queue_idx = nvme_tcp_queue_id(queue);  in nvme_tcp_tagset()
259 		return queue->ctrl->admin_tag_set.tags[queue_idx];  in nvme_tcp_tagset()
260 	return queue->ctrl->tag_set.tags[queue_idx - 1];  in nvme_tcp_tagset()
263 static inline u8 nvme_tcp_hdgst_len(struct nvme_tcp_queue *queue)  in nvme_tcp_hdgst_len()  argument
265 	return queue->hdr_digest ? NVME_TCP_DIGEST_LENGTH : 0;  in nvme_tcp_hdgst_len()
268 static inline u8 nvme_tcp_ddgst_len(struct nvme_tcp_queue *queue)  in nvme_tcp_ddgst_len()  argument
270 	return queue->data_digest ? NVME_TCP_DIGEST_LENGTH : 0;  in nvme_tcp_ddgst_len()
289 	return req->queue->cmnd_capsule_len - sizeof(struct nvme_command);  in nvme_tcp_inline_data_size()
294 	return req == &req->queue->ctrl->async_req;  in nvme_tcp_async_req()
383 static inline void nvme_tcp_send_all(struct nvme_tcp_queue *queue)  in nvme_tcp_send_all()  argument
389 		ret = nvme_tcp_try_send(queue);  in nvme_tcp_send_all()
393 static inline bool nvme_tcp_queue_has_pending(struct nvme_tcp_queue *queue)  in nvme_tcp_queue_has_pending()  argument
395 	return !list_empty(&queue->send_list) ||  in nvme_tcp_queue_has_pending()
396 		!llist_empty(&queue->req_list);  in nvme_tcp_queue_has_pending()
399 static inline bool nvme_tcp_queue_more(struct nvme_tcp_queue *queue)  in nvme_tcp_queue_more()  argument
401 	return !nvme_tcp_queue_tls(queue) &&  in nvme_tcp_queue_more()
402 		nvme_tcp_queue_has_pending(queue);  in nvme_tcp_queue_more()
408 	struct nvme_tcp_queue *queue = req->queue;  in nvme_tcp_queue_request()  local
411 	empty = llist_add(&req->lentry, &queue->req_list) &&  in nvme_tcp_queue_request()
412 		list_empty(&queue->send_list) && !queue->request;  in nvme_tcp_queue_request()
419 	if (queue->io_cpu == raw_smp_processor_id() &&  in nvme_tcp_queue_request()
420 	    empty && mutex_trylock(&queue->send_mutex)) {  in nvme_tcp_queue_request()
421 		nvme_tcp_send_all(queue);  in nvme_tcp_queue_request()
422 		mutex_unlock(&queue->send_mutex);  in nvme_tcp_queue_request()
425 	if (last && nvme_tcp_queue_has_pending(queue))  in nvme_tcp_queue_request()
426 		queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);  in nvme_tcp_queue_request()
429 static void nvme_tcp_process_req_list(struct nvme_tcp_queue *queue)  in nvme_tcp_process_req_list()  argument
434 	for (node = llist_del_all(&queue->req_list); node; node = node->next) {  in nvme_tcp_process_req_list()
436 		list_add(&req->entry, &queue->send_list);  in nvme_tcp_process_req_list()
441 nvme_tcp_fetch_request(struct nvme_tcp_queue *queue)  in nvme_tcp_fetch_request()  argument
445 	req = list_first_entry_or_null(&queue->send_list,  in nvme_tcp_fetch_request()
448 		nvme_tcp_process_req_list(queue);  in nvme_tcp_fetch_request()
449 		req = list_first_entry_or_null(&queue->send_list,  in nvme_tcp_fetch_request()
494 static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue,  in nvme_tcp_verify_hdgst()  argument
502 		dev_err(queue->ctrl->ctrl.device,  in nvme_tcp_verify_hdgst()
504 			nvme_tcp_queue_id(queue));  in nvme_tcp_verify_hdgst()
511 		dev_err(queue->ctrl->ctrl.device,  in nvme_tcp_verify_hdgst()
520 static int nvme_tcp_check_ddgst(struct nvme_tcp_queue *queue, void *pdu)  in nvme_tcp_check_ddgst()  argument
523 	u8 digest_len = nvme_tcp_hdgst_len(queue);  in nvme_tcp_check_ddgst()
530 		dev_err(queue->ctrl->ctrl.device,  in nvme_tcp_check_ddgst()
532 		nvme_tcp_queue_id(queue));  in nvme_tcp_check_ddgst()
535 	queue->rcv_crc = NVME_TCP_CRC_SEED;  in nvme_tcp_check_ddgst()
556 	struct nvme_tcp_queue *queue = &ctrl->queues[queue_idx];  in nvme_tcp_init_request()  local
557 	u8 hdgst = nvme_tcp_hdgst_len(queue);  in nvme_tcp_init_request()
559 	req->pdu = page_frag_alloc(&queue->pf_cache,  in nvme_tcp_init_request()
566 	req->queue = queue;  in nvme_tcp_init_request()
579 	struct nvme_tcp_queue *queue = &ctrl->queues[hctx_idx + 1];  in nvme_tcp_init_hctx()  local
581 	hctx->driver_data = queue;  in nvme_tcp_init_hctx()
589 	struct nvme_tcp_queue *queue = &ctrl->queues[0];  in nvme_tcp_init_admin_hctx()  local
591 	hctx->driver_data = queue;  in nvme_tcp_init_admin_hctx()
596 nvme_tcp_recv_state(struct nvme_tcp_queue *queue)  in nvme_tcp_recv_state()  argument
598 	return  (queue->pdu_remaining) ? NVME_TCP_RECV_PDU :  in nvme_tcp_recv_state()
599 		(queue->ddgst_remaining) ? NVME_TCP_RECV_DDGST :  in nvme_tcp_recv_state()
603 static void nvme_tcp_init_recv_ctx(struct nvme_tcp_queue *queue)  in nvme_tcp_init_recv_ctx()  argument
605 	queue->pdu_remaining = sizeof(struct nvme_tcp_rsp_pdu) +  in nvme_tcp_init_recv_ctx()
606 				nvme_tcp_hdgst_len(queue);  in nvme_tcp_init_recv_ctx()
607 	queue->pdu_offset = 0;  in nvme_tcp_init_recv_ctx()
608 	queue->data_remaining = -1;  in nvme_tcp_init_recv_ctx()
609 	queue->ddgst_remaining = 0;  in nvme_tcp_init_recv_ctx()
621 static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue,  in nvme_tcp_process_nvme_cqe()  argument
627 	rq = nvme_find_rq(nvme_tcp_tagset(queue), cqe->command_id);  in nvme_tcp_process_nvme_cqe()
629 		dev_err(queue->ctrl->ctrl.device,  in nvme_tcp_process_nvme_cqe()
631 			cqe->command_id, nvme_tcp_queue_id(queue));  in nvme_tcp_process_nvme_cqe()
632 		nvme_tcp_error_recovery(&queue->ctrl->ctrl);  in nvme_tcp_process_nvme_cqe()
642 	queue->nr_cqe++;  in nvme_tcp_process_nvme_cqe()
647 static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue,  in nvme_tcp_handle_c2h_data()  argument
652 	rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id);  in nvme_tcp_handle_c2h_data()
654 		dev_err(queue->ctrl->ctrl.device,  in nvme_tcp_handle_c2h_data()
656 			pdu->command_id, nvme_tcp_queue_id(queue));  in nvme_tcp_handle_c2h_data()
661 		dev_err(queue->ctrl->ctrl.device,  in nvme_tcp_handle_c2h_data()
663 			nvme_tcp_queue_id(queue), rq->tag);  in nvme_tcp_handle_c2h_data()
667 	queue->data_remaining = le32_to_cpu(pdu->data_length);  in nvme_tcp_handle_c2h_data()
671 		dev_err(queue->ctrl->ctrl.device,  in nvme_tcp_handle_c2h_data()
673 			nvme_tcp_queue_id(queue), rq->tag);  in nvme_tcp_handle_c2h_data()
674 		nvme_tcp_error_recovery(&queue->ctrl->ctrl);  in nvme_tcp_handle_c2h_data()
681 static int nvme_tcp_handle_comp(struct nvme_tcp_queue *queue,  in nvme_tcp_handle_comp()  argument
693 	if (unlikely(nvme_is_aen_req(nvme_tcp_queue_id(queue),  in nvme_tcp_handle_comp()
695 		nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,  in nvme_tcp_handle_comp()
698 		ret = nvme_tcp_process_nvme_cqe(queue, cqe);  in nvme_tcp_handle_comp()
706 	struct nvme_tcp_queue *queue = req->queue;  in nvme_tcp_setup_h2c_data_pdu()  local
709 	u8 hdgst = nvme_tcp_hdgst_len(queue);  in nvme_tcp_setup_h2c_data_pdu()
710 	u8 ddgst = nvme_tcp_ddgst_len(queue);  in nvme_tcp_setup_h2c_data_pdu()
714 	req->pdu_len = min(req->h2cdata_left, queue->maxh2cdata);  in nvme_tcp_setup_h2c_data_pdu()
723 	if (queue->hdr_digest)  in nvme_tcp_setup_h2c_data_pdu()
725 	if (queue->data_digest)  in nvme_tcp_setup_h2c_data_pdu()
737 static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue,  in nvme_tcp_handle_r2t()  argument
745 	rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id);  in nvme_tcp_handle_r2t()
747 		dev_err(queue->ctrl->ctrl.device,  in nvme_tcp_handle_r2t()
749 			pdu->command_id, nvme_tcp_queue_id(queue));  in nvme_tcp_handle_r2t()
755 		dev_err(queue->ctrl->ctrl.device,  in nvme_tcp_handle_r2t()
762 		dev_err(queue->ctrl->ctrl.device,  in nvme_tcp_handle_r2t()
769 		dev_err(queue->ctrl->ctrl.device,  in nvme_tcp_handle_r2t()
777 		dev_err(queue->ctrl->ctrl.device,  in nvme_tcp_handle_r2t()
790 	llist_add(&req->lentry, &queue->req_list);  in nvme_tcp_handle_r2t()
791 	queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);  in nvme_tcp_handle_r2t()
796 static void nvme_tcp_handle_c2h_term(struct nvme_tcp_queue *queue,  in nvme_tcp_handle_c2h_term()  argument
814 		dev_err(queue->ctrl->ctrl.device,  in nvme_tcp_handle_c2h_term()
826 	dev_err(queue->ctrl->ctrl.device,  in nvme_tcp_handle_c2h_term()
830 static int nvme_tcp_recv_pdu(struct nvme_tcp_queue *queue, struct sk_buff *skb,  in nvme_tcp_recv_pdu()  argument
834 	char *pdu = queue->pdu;  in nvme_tcp_recv_pdu()
835 	size_t rcv_len = min_t(size_t, *len, queue->pdu_remaining);  in nvme_tcp_recv_pdu()
839 		&pdu[queue->pdu_offset], rcv_len);  in nvme_tcp_recv_pdu()
843 	queue->pdu_remaining -= rcv_len;  in nvme_tcp_recv_pdu()
844 	queue->pdu_offset += rcv_len;  in nvme_tcp_recv_pdu()
847 	if (queue->pdu_remaining)  in nvme_tcp_recv_pdu()
850 	hdr = queue->pdu;  in nvme_tcp_recv_pdu()
855 		dev_err(queue->ctrl->ctrl.device,  in nvme_tcp_recv_pdu()
866 		nvme_tcp_handle_c2h_term(queue, (void *)queue->pdu);  in nvme_tcp_recv_pdu()
870 	if (queue->hdr_digest) {  in nvme_tcp_recv_pdu()
871 		ret = nvme_tcp_verify_hdgst(queue, queue->pdu, hdr->hlen);  in nvme_tcp_recv_pdu()
877 	if (queue->data_digest) {  in nvme_tcp_recv_pdu()
878 		ret = nvme_tcp_check_ddgst(queue, queue->pdu);  in nvme_tcp_recv_pdu()
885 		return nvme_tcp_handle_c2h_data(queue, (void *)queue->pdu);  in nvme_tcp_recv_pdu()
887 		nvme_tcp_init_recv_ctx(queue);  in nvme_tcp_recv_pdu()
888 		return nvme_tcp_handle_comp(queue, (void *)queue->pdu);  in nvme_tcp_recv_pdu()
890 		nvme_tcp_init_recv_ctx(queue);  in nvme_tcp_recv_pdu()
891 		return nvme_tcp_handle_r2t(queue, (void *)queue->pdu);  in nvme_tcp_recv_pdu()
897 	dev_err(queue->ctrl->ctrl.device,  in nvme_tcp_recv_pdu()
910 static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb,  in nvme_tcp_recv_data()  argument
913 	struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;  in nvme_tcp_recv_data()
915 		nvme_cid_to_rq(nvme_tcp_tagset(queue), pdu->command_id);  in nvme_tcp_recv_data()
921 		recv_len = min_t(size_t, *len, queue->data_remaining);  in nvme_tcp_recv_data()
933 				dev_err(queue->ctrl->ctrl.device,  in nvme_tcp_recv_data()
935 					nvme_tcp_queue_id(queue), rq->tag);  in nvme_tcp_recv_data()
936 				nvme_tcp_init_recv_ctx(queue);  in nvme_tcp_recv_data()
946 		if (queue->data_digest)  in nvme_tcp_recv_data()
948 				&req->iter, recv_len, &queue->rcv_crc);  in nvme_tcp_recv_data()
953 			dev_err(queue->ctrl->ctrl.device,  in nvme_tcp_recv_data()
955 				nvme_tcp_queue_id(queue), rq->tag);  in nvme_tcp_recv_data()
961 		queue->data_remaining -= recv_len;  in nvme_tcp_recv_data()
964 	if (!queue->data_remaining) {  in nvme_tcp_recv_data()
965 		if (queue->data_digest) {  in nvme_tcp_recv_data()
966 			queue->exp_ddgst = nvme_tcp_ddgst_final(queue->rcv_crc);  in nvme_tcp_recv_data()
967 			queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH;  in nvme_tcp_recv_data()
972 				queue->nr_cqe++;  in nvme_tcp_recv_data()
974 			nvme_tcp_init_recv_ctx(queue);  in nvme_tcp_recv_data()
981 static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue *queue,  in nvme_tcp_recv_ddgst()  argument
984 	struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;  in nvme_tcp_recv_ddgst()
985 	char *ddgst = (char *)&queue->recv_ddgst;  in nvme_tcp_recv_ddgst()
986 	size_t recv_len = min_t(size_t, *len, queue->ddgst_remaining);  in nvme_tcp_recv_ddgst()
987 	off_t off = NVME_TCP_DIGEST_LENGTH - queue->ddgst_remaining;  in nvme_tcp_recv_ddgst()
994 	queue->ddgst_remaining -= recv_len;  in nvme_tcp_recv_ddgst()
997 	if (queue->ddgst_remaining)  in nvme_tcp_recv_ddgst()
1000 	if (queue->recv_ddgst != queue->exp_ddgst) {  in nvme_tcp_recv_ddgst()
1001 		struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue),  in nvme_tcp_recv_ddgst()
1007 		dev_err(queue->ctrl->ctrl.device,  in nvme_tcp_recv_ddgst()
1009 			le32_to_cpu(queue->recv_ddgst),  in nvme_tcp_recv_ddgst()
1010 			le32_to_cpu(queue->exp_ddgst));  in nvme_tcp_recv_ddgst()
1014 		struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue),  in nvme_tcp_recv_ddgst()
1019 		queue->nr_cqe++;  in nvme_tcp_recv_ddgst()
1022 	nvme_tcp_init_recv_ctx(queue);  in nvme_tcp_recv_ddgst()
1029 	struct nvme_tcp_queue *queue = desc->arg.data;  in nvme_tcp_recv_skb()  local
1033 	if (unlikely(!queue->rd_enabled))  in nvme_tcp_recv_skb()
1037 		switch (nvme_tcp_recv_state(queue)) {  in nvme_tcp_recv_skb()
1039 			result = nvme_tcp_recv_pdu(queue, skb, &offset, &len);  in nvme_tcp_recv_skb()
1042 			result = nvme_tcp_recv_data(queue, skb, &offset, &len);  in nvme_tcp_recv_skb()
1045 			result = nvme_tcp_recv_ddgst(queue, skb, &offset, &len);  in nvme_tcp_recv_skb()
1051 			dev_err(queue->ctrl->ctrl.device,  in nvme_tcp_recv_skb()
1053 			queue->rd_enabled = false;  in nvme_tcp_recv_skb()
1054 			nvme_tcp_error_recovery(&queue->ctrl->ctrl);  in nvme_tcp_recv_skb()
1064 	struct nvme_tcp_queue *queue;  in nvme_tcp_data_ready()  local
1069 	queue = sk->sk_user_data;  in nvme_tcp_data_ready()
1070 	if (likely(queue && queue->rd_enabled) &&  in nvme_tcp_data_ready()
1071 	    !test_bit(NVME_TCP_Q_POLLING, &queue->flags))  in nvme_tcp_data_ready()
1072 		queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);  in nvme_tcp_data_ready()
1078 	struct nvme_tcp_queue *queue;  in nvme_tcp_write_space()  local
1081 	queue = sk->sk_user_data;  in nvme_tcp_write_space()
1082 	if (likely(queue && sk_stream_is_writeable(sk))) {  in nvme_tcp_write_space()
1085 		if (nvme_tcp_queue_tls(queue))  in nvme_tcp_write_space()
1086 			queue->write_space(sk);  in nvme_tcp_write_space()
1087 		queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);  in nvme_tcp_write_space()
1094 	struct nvme_tcp_queue *queue;  in nvme_tcp_state_change()  local
1097 	queue = sk->sk_user_data;  in nvme_tcp_state_change()
1098 	if (!queue)  in nvme_tcp_state_change()
1107 		nvme_tcp_error_recovery(&queue->ctrl->ctrl);  in nvme_tcp_state_change()
1110 		dev_info(queue->ctrl->ctrl.device,  in nvme_tcp_state_change()
1112 			nvme_tcp_queue_id(queue), sk->sk_state);  in nvme_tcp_state_change()
1115 	queue->state_change(sk);  in nvme_tcp_state_change()
1120 static inline void nvme_tcp_done_send_req(struct nvme_tcp_queue *queue)  in nvme_tcp_done_send_req()  argument
1122 	queue->request = NULL;  in nvme_tcp_done_send_req()
1130 		nvme_complete_async_event(&req->queue->ctrl->ctrl,  in nvme_tcp_fail_request()
1140 	struct nvme_tcp_queue *queue = req->queue;  in nvme_tcp_try_send_data()  local
1156 		if (last && !queue->data_digest && !nvme_tcp_queue_more(queue))  in nvme_tcp_try_send_data()
1166 		ret = sock_sendmsg(queue->sock, &msg);  in nvme_tcp_try_send_data()
1170 		if (queue->data_digest)  in nvme_tcp_try_send_data()
1171 			nvme_tcp_ddgst_update(&queue->snd_crc, page,  in nvme_tcp_try_send_data()
1184 			if (queue->data_digest) {  in nvme_tcp_try_send_data()
1186 					nvme_tcp_ddgst_final(queue->snd_crc);  in nvme_tcp_try_send_data()
1193 					nvme_tcp_done_send_req(queue);  in nvme_tcp_try_send_data()
1203 	struct nvme_tcp_queue *queue = req->queue;  in nvme_tcp_try_send_cmd_pdu()  local
1208 	u8 hdgst = nvme_tcp_hdgst_len(queue);  in nvme_tcp_try_send_cmd_pdu()
1212 	if (inline_data || nvme_tcp_queue_more(queue))  in nvme_tcp_try_send_cmd_pdu()
1217 	if (queue->hdr_digest && !req->offset)  in nvme_tcp_try_send_cmd_pdu()
1222 	ret = sock_sendmsg(queue->sock, &msg);  in nvme_tcp_try_send_cmd_pdu()
1230 			if (queue->data_digest)  in nvme_tcp_try_send_cmd_pdu()
1231 				queue->snd_crc = NVME_TCP_CRC_SEED;  in nvme_tcp_try_send_cmd_pdu()
1233 			nvme_tcp_done_send_req(queue);  in nvme_tcp_try_send_cmd_pdu()
1244 	struct nvme_tcp_queue *queue = req->queue;  in nvme_tcp_try_send_data_pdu()  local
1248 	u8 hdgst = nvme_tcp_hdgst_len(queue);  in nvme_tcp_try_send_data_pdu()
1252 	if (queue->hdr_digest && !req->offset)  in nvme_tcp_try_send_data_pdu()
1260 	ret = sock_sendmsg(queue->sock, &msg);  in nvme_tcp_try_send_data_pdu()
1267 		if (queue->data_digest)  in nvme_tcp_try_send_data_pdu()
1268 			queue->snd_crc = NVME_TCP_CRC_SEED;  in nvme_tcp_try_send_data_pdu()
1278 	struct nvme_tcp_queue *queue = req->queue;  in nvme_tcp_try_send_ddgst()  local
1288 	if (nvme_tcp_queue_more(queue))  in nvme_tcp_try_send_ddgst()
1293 	ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);  in nvme_tcp_try_send_ddgst()
1301 			nvme_tcp_done_send_req(queue);  in nvme_tcp_try_send_ddgst()
1309 static int nvme_tcp_try_send(struct nvme_tcp_queue *queue)  in nvme_tcp_try_send()  argument
1315 	if (!queue->request) {  in nvme_tcp_try_send()
1316 		queue->request = nvme_tcp_fetch_request(queue);  in nvme_tcp_try_send()
1317 		if (!queue->request)  in nvme_tcp_try_send()
1320 	req = queue->request;  in nvme_tcp_try_send()
1349 		dev_err(queue->ctrl->ctrl.device,  in nvme_tcp_try_send()
1351 		nvme_tcp_fail_request(queue->request);  in nvme_tcp_try_send()
1352 		nvme_tcp_done_send_req(queue);  in nvme_tcp_try_send()
1359 static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue)  in nvme_tcp_try_recv()  argument
1361 	struct socket *sock = queue->sock;  in nvme_tcp_try_recv()
1366 	rd_desc.arg.data = queue;  in nvme_tcp_try_recv()
1369 	queue->nr_cqe = 0;  in nvme_tcp_try_recv()
1377 	struct nvme_tcp_queue *queue =  in nvme_tcp_io_work()  local
1385 		if (mutex_trylock(&queue->send_mutex)) {  in nvme_tcp_io_work()
1386 			result = nvme_tcp_try_send(queue);  in nvme_tcp_io_work()
1387 			mutex_unlock(&queue->send_mutex);  in nvme_tcp_io_work()
1394 		result = nvme_tcp_try_recv(queue);  in nvme_tcp_io_work()
1401 		if (nvme_tcp_queue_has_pending(queue) &&  in nvme_tcp_io_work()
1402 		    sk_stream_is_writeable(queue->sock->sk))  in nvme_tcp_io_work()
1405 		if (!pending || !queue->rd_enabled)  in nvme_tcp_io_work()
1410 	queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);  in nvme_tcp_io_work()
1422 	struct nvme_tcp_queue *queue = &ctrl->queues[0];  in nvme_tcp_alloc_async_req()  local
1424 	u8 hdgst = nvme_tcp_hdgst_len(queue);  in nvme_tcp_alloc_async_req()
1426 	async->pdu = page_frag_alloc(&queue->pf_cache,  in nvme_tcp_alloc_async_req()
1432 	async->queue = &ctrl->queues[0];  in nvme_tcp_alloc_async_req()
1439 	struct nvme_tcp_queue *queue = &ctrl->queues[qid];  in nvme_tcp_free_queue()  local
1442 	if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))  in nvme_tcp_free_queue()
1445 	page_frag_cache_drain(&queue->pf_cache);  in nvme_tcp_free_queue()
1449 	fput(queue->sock->file);  in nvme_tcp_free_queue()
1450 	queue->sock = NULL;  in nvme_tcp_free_queue()
1453 	kfree(queue->pdu);  in nvme_tcp_free_queue()
1454 	mutex_destroy(&queue->send_mutex);  in nvme_tcp_free_queue()
1455 	mutex_destroy(&queue->queue_lock);  in nvme_tcp_free_queue()
1458 static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue)  in nvme_tcp_init_connection()  argument
1487 	if (queue->hdr_digest)  in nvme_tcp_init_connection()
1489 	if (queue->data_digest)  in nvme_tcp_init_connection()
1494 	ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);  in nvme_tcp_init_connection()
1497 			nvme_tcp_queue_id(queue), ret);  in nvme_tcp_init_connection()
1504 	if (nvme_tcp_queue_tls(queue)) {  in nvme_tcp_init_connection()
1509 	ret = kernel_recvmsg(queue->sock, &msg, &iov, 1,  in nvme_tcp_init_connection()
1515 			nvme_tcp_queue_id(queue), ret);  in nvme_tcp_init_connection()
1519 	if (nvme_tcp_queue_tls(queue)) {  in nvme_tcp_init_connection()
1520 		ctype = tls_get_record_type(queue->sock->sk,  in nvme_tcp_init_connection()
1524 			       nvme_tcp_queue_id(queue), ctype);  in nvme_tcp_init_connection()
1531 			nvme_tcp_queue_id(queue), icresp->hdr.type);  in nvme_tcp_init_connection()
1537 			nvme_tcp_queue_id(queue), icresp->hdr.plen);  in nvme_tcp_init_connection()
1543 			nvme_tcp_queue_id(queue), icresp->pfv);  in nvme_tcp_init_connection()
1548 	if ((queue->data_digest && !ctrl_ddgst) ||  in nvme_tcp_init_connection()
1549 	    (!queue->data_digest && ctrl_ddgst)) {  in nvme_tcp_init_connection()
1551 			nvme_tcp_queue_id(queue),  in nvme_tcp_init_connection()
1552 			queue->data_digest ? "enabled" : "disabled",  in nvme_tcp_init_connection()
1558 	if ((queue->hdr_digest && !ctrl_hdgst) ||  in nvme_tcp_init_connection()
1559 	    (!queue->hdr_digest && ctrl_hdgst)) {  in nvme_tcp_init_connection()
1561 			nvme_tcp_queue_id(queue),  in nvme_tcp_init_connection()
1562 			queue->hdr_digest ? "enabled" : "disabled",  in nvme_tcp_init_connection()
1569 			nvme_tcp_queue_id(queue), icresp->cpda);  in nvme_tcp_init_connection()
1576 		       nvme_tcp_queue_id(queue), maxh2cdata);  in nvme_tcp_init_connection()
1579 	queue->maxh2cdata = maxh2cdata;  in nvme_tcp_init_connection()
1589 static bool nvme_tcp_admin_queue(struct nvme_tcp_queue *queue)  in nvme_tcp_admin_queue()  argument
1591 	return nvme_tcp_queue_id(queue) == 0;  in nvme_tcp_admin_queue()
1594 static bool nvme_tcp_default_queue(struct nvme_tcp_queue *queue)  in nvme_tcp_default_queue()  argument
1596 	struct nvme_tcp_ctrl *ctrl = queue->ctrl;  in nvme_tcp_default_queue()
1597 	int qid = nvme_tcp_queue_id(queue);  in nvme_tcp_default_queue()
1599 	return !nvme_tcp_admin_queue(queue) &&  in nvme_tcp_default_queue()
1603 static bool nvme_tcp_read_queue(struct nvme_tcp_queue *queue)  in nvme_tcp_read_queue()  argument
1605 	struct nvme_tcp_ctrl *ctrl = queue->ctrl;  in nvme_tcp_read_queue()
1606 	int qid = nvme_tcp_queue_id(queue);  in nvme_tcp_read_queue()
1608 	return !nvme_tcp_admin_queue(queue) &&  in nvme_tcp_read_queue()
1609 		!nvme_tcp_default_queue(queue) &&  in nvme_tcp_read_queue()
1614 static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue)  in nvme_tcp_poll_queue()  argument
1616 	struct nvme_tcp_ctrl *ctrl = queue->ctrl;  in nvme_tcp_poll_queue()
1617 	int qid = nvme_tcp_queue_id(queue);  in nvme_tcp_poll_queue()
1619 	return !nvme_tcp_admin_queue(queue) &&  in nvme_tcp_poll_queue()
1620 		!nvme_tcp_default_queue(queue) &&  in nvme_tcp_poll_queue()
1621 		!nvme_tcp_read_queue(queue) &&  in nvme_tcp_poll_queue()
1636 static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue)  in nvme_tcp_set_queue_io_cpu()  argument
1638 	struct nvme_tcp_ctrl *ctrl = queue->ctrl;  in nvme_tcp_set_queue_io_cpu()
1640 	int qid = nvme_tcp_queue_id(queue) - 1;  in nvme_tcp_set_queue_io_cpu()
1647 	if (nvme_tcp_default_queue(queue))  in nvme_tcp_set_queue_io_cpu()
1649 	else if (nvme_tcp_read_queue(queue))  in nvme_tcp_set_queue_io_cpu()
1651 	else if (nvme_tcp_poll_queue(queue))  in nvme_tcp_set_queue_io_cpu()
1670 		queue->io_cpu = io_cpu;  in nvme_tcp_set_queue_io_cpu()
1672 		set_bit(NVME_TCP_Q_IO_CPU_SET, &queue->flags);  in nvme_tcp_set_queue_io_cpu()
1676 		qid, queue->io_cpu);  in nvme_tcp_set_queue_io_cpu()
1681 	struct nvme_tcp_queue *queue = data;  in nvme_tcp_tls_done()  local
1682 	struct nvme_tcp_ctrl *ctrl = queue->ctrl;  in nvme_tcp_tls_done()
1683 	int qid = nvme_tcp_queue_id(queue);  in nvme_tcp_tls_done()
1690 		queue->tls_err = -status;  in nvme_tcp_tls_done()
1698 		queue->tls_err = -ENOKEY;  in nvme_tcp_tls_done()
1700 		queue->tls_enabled = true;  in nvme_tcp_tls_done()
1704 		queue->tls_err = 0;  in nvme_tcp_tls_done()
1708 	complete(&queue->tls_complete);  in nvme_tcp_tls_done()
1712 			      struct nvme_tcp_queue *queue,  in nvme_tcp_start_tls()  argument
1715 	int qid = nvme_tcp_queue_id(queue);  in nvme_tcp_start_tls()
1724 	args.ta_sock = queue->sock;  in nvme_tcp_start_tls()
1726 	args.ta_data = queue;  in nvme_tcp_start_tls()
1733 	queue->tls_err = -EOPNOTSUPP;  in nvme_tcp_start_tls()
1734 	init_completion(&queue->tls_complete);  in nvme_tcp_start_tls()
1741 	ret = wait_for_completion_interruptible_timeout(&queue->tls_complete, tmo);  in nvme_tcp_start_tls()
1749 		tls_handshake_cancel(queue->sock->sk);  in nvme_tcp_start_tls()
1751 		if (queue->tls_err) {  in nvme_tcp_start_tls()
1754 				qid, queue->tls_err);  in nvme_tcp_start_tls()
1759 		ret = queue->tls_err;  in nvme_tcp_start_tls()
1768 	struct nvme_tcp_queue *queue = &ctrl->queues[qid];  in nvme_tcp_alloc_queue()  local
1772 	mutex_init(&queue->queue_lock);  in nvme_tcp_alloc_queue()
1773 	queue->ctrl = ctrl;  in nvme_tcp_alloc_queue()
1774 	init_llist_head(&queue->req_list);  in nvme_tcp_alloc_queue()
1775 	INIT_LIST_HEAD(&queue->send_list);  in nvme_tcp_alloc_queue()
1776 	mutex_init(&queue->send_mutex);  in nvme_tcp_alloc_queue()
1777 	INIT_WORK(&queue->io_work, nvme_tcp_io_work);  in nvme_tcp_alloc_queue()
1780 		queue->cmnd_capsule_len = nctrl->ioccsz * 16;  in nvme_tcp_alloc_queue()
1782 		queue->cmnd_capsule_len = sizeof(struct nvme_command) +  in nvme_tcp_alloc_queue()
1787 			IPPROTO_TCP, &queue->sock);  in nvme_tcp_alloc_queue()
1794 	sock_file = sock_alloc_file(queue->sock, O_CLOEXEC, NULL);  in nvme_tcp_alloc_queue()
1800 	sk_net_refcnt_upgrade(queue->sock->sk);  in nvme_tcp_alloc_queue()
1801 	nvme_tcp_reclassify_socket(queue->sock);  in nvme_tcp_alloc_queue()
1804 	tcp_sock_set_syncnt(queue->sock->sk, 1);  in nvme_tcp_alloc_queue()
1807 	tcp_sock_set_nodelay(queue->sock->sk);  in nvme_tcp_alloc_queue()
1814 	sock_no_linger(queue->sock->sk);  in nvme_tcp_alloc_queue()
1817 		sock_set_priority(queue->sock->sk, so_priority);  in nvme_tcp_alloc_queue()
1821 		ip_sock_set_tos(queue->sock->sk, nctrl->opts->tos);  in nvme_tcp_alloc_queue()
1824 	queue->sock->sk->sk_rcvtimeo = 10 * HZ;  in nvme_tcp_alloc_queue()
1826 	queue->sock->sk->sk_allocation = GFP_ATOMIC;  in nvme_tcp_alloc_queue()
1827 	queue->sock->sk->sk_use_task_frag = false;  in nvme_tcp_alloc_queue()
1828 	queue->io_cpu = WORK_CPU_UNBOUND;  in nvme_tcp_alloc_queue()
1829 	queue->request = NULL;  in nvme_tcp_alloc_queue()
1830 	queue->data_remaining = 0;  in nvme_tcp_alloc_queue()
1831 	queue->ddgst_remaining = 0;  in nvme_tcp_alloc_queue()
1832 	queue->pdu_remaining = 0;  in nvme_tcp_alloc_queue()
1833 	queue->pdu_offset = 0;  in nvme_tcp_alloc_queue()
1834 	sk_set_memalloc(queue->sock->sk);  in nvme_tcp_alloc_queue()
1837 		ret = kernel_bind(queue->sock, (struct sockaddr_unsized *)&ctrl->src_addr,  in nvme_tcp_alloc_queue()
1851 		ret = sock_setsockopt(queue->sock, SOL_SOCKET, SO_BINDTODEVICE,  in nvme_tcp_alloc_queue()
1861 	queue->hdr_digest = nctrl->opts->hdr_digest;  in nvme_tcp_alloc_queue()
1862 	queue->data_digest = nctrl->opts->data_digest;  in nvme_tcp_alloc_queue()
1865 			nvme_tcp_hdgst_len(queue);  in nvme_tcp_alloc_queue()
1866 	queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL);  in nvme_tcp_alloc_queue()
1867 	if (!queue->pdu) {  in nvme_tcp_alloc_queue()
1873 			nvme_tcp_queue_id(queue));  in nvme_tcp_alloc_queue()
1875 	ret = kernel_connect(queue->sock, (struct sockaddr_unsized *)&ctrl->addr,  in nvme_tcp_alloc_queue()
1885 		ret = nvme_tcp_start_tls(nctrl, queue, pskid);  in nvme_tcp_alloc_queue()
1890 	ret = nvme_tcp_init_connection(queue);  in nvme_tcp_alloc_queue()
1894 	set_bit(NVME_TCP_Q_ALLOCATED, &queue->flags);  in nvme_tcp_alloc_queue()
1899 	kernel_sock_shutdown(queue->sock, SHUT_RDWR);  in nvme_tcp_alloc_queue()
1901 	kfree(queue->pdu);  in nvme_tcp_alloc_queue()
1904 	fput(queue->sock->file);  in nvme_tcp_alloc_queue()
1905 	queue->sock = NULL;  in nvme_tcp_alloc_queue()
1907 	mutex_destroy(&queue->send_mutex);  in nvme_tcp_alloc_queue()
1908 	mutex_destroy(&queue->queue_lock);  in nvme_tcp_alloc_queue()
1912 static void nvme_tcp_restore_sock_ops(struct nvme_tcp_queue *queue)  in nvme_tcp_restore_sock_ops()  argument
1914 	struct socket *sock = queue->sock;  in nvme_tcp_restore_sock_ops()
1918 	sock->sk->sk_data_ready = queue->data_ready;  in nvme_tcp_restore_sock_ops()
1919 	sock->sk->sk_state_change = queue->state_change;  in nvme_tcp_restore_sock_ops()
1920 	sock->sk->sk_write_space  = queue->write_space;  in nvme_tcp_restore_sock_ops()
1924 static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue)  in __nvme_tcp_stop_queue()  argument
1926 	kernel_sock_shutdown(queue->sock, SHUT_RDWR);  in __nvme_tcp_stop_queue()
1927 	nvme_tcp_restore_sock_ops(queue);  in __nvme_tcp_stop_queue()
1928 	cancel_work_sync(&queue->io_work);  in __nvme_tcp_stop_queue()
1934 	struct nvme_tcp_queue *queue = &ctrl->queues[qid];  in nvme_tcp_stop_queue_nowait()  local
1936 	if (!test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))  in nvme_tcp_stop_queue_nowait()
1939 	if (test_and_clear_bit(NVME_TCP_Q_IO_CPU_SET, &queue->flags))  in nvme_tcp_stop_queue_nowait()
1940 		atomic_dec(&nvme_tcp_cpu_queues[queue->io_cpu]);  in nvme_tcp_stop_queue_nowait()
1942 	mutex_lock(&queue->queue_lock);  in nvme_tcp_stop_queue_nowait()
1943 	if (test_and_clear_bit(NVME_TCP_Q_LIVE, &queue->flags))  in nvme_tcp_stop_queue_nowait()
1944 		__nvme_tcp_stop_queue(queue);  in nvme_tcp_stop_queue_nowait()
1946 	queue->tls_enabled = false;  in nvme_tcp_stop_queue_nowait()
1947 	mutex_unlock(&queue->queue_lock);  in nvme_tcp_stop_queue_nowait()
1953 	struct nvme_tcp_queue *queue = &ctrl->queues[qid];  in nvme_tcp_wait_queue()  local
1957 		if (!test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags) ||  in nvme_tcp_wait_queue()
1958 		    !sk_wmem_alloc_get(queue->sock->sk))  in nvme_tcp_wait_queue()
1975 static void nvme_tcp_setup_sock_ops(struct nvme_tcp_queue *queue)  in nvme_tcp_setup_sock_ops()  argument
1977 	write_lock_bh(&queue->sock->sk->sk_callback_lock);  in nvme_tcp_setup_sock_ops()
1978 	queue->sock->sk->sk_user_data = queue;  in nvme_tcp_setup_sock_ops()
1979 	queue->state_change = queue->sock->sk->sk_state_change;  in nvme_tcp_setup_sock_ops()
1980 	queue->data_ready = queue->sock->sk->sk_data_ready;  in nvme_tcp_setup_sock_ops()
1981 	queue->write_space = queue->sock->sk->sk_write_space;  in nvme_tcp_setup_sock_ops()
1982 	queue->sock->sk->sk_data_ready = nvme_tcp_data_ready;  in nvme_tcp_setup_sock_ops()
1983 	queue->sock->sk->sk_state_change = nvme_tcp_state_change;  in nvme_tcp_setup_sock_ops()
1984 	queue->sock->sk->sk_write_space = nvme_tcp_write_space;  in nvme_tcp_setup_sock_ops()
1986 	queue->sock->sk->sk_ll_usec = 1;  in nvme_tcp_setup_sock_ops()
1988 	write_unlock_bh(&queue->sock->sk->sk_callback_lock);  in nvme_tcp_setup_sock_ops()
1994 	struct nvme_tcp_queue *queue = &ctrl->queues[idx];  in nvme_tcp_start_queue()  local
1997 	queue->rd_enabled = true;  in nvme_tcp_start_queue()
1998 	nvme_tcp_init_recv_ctx(queue);  in nvme_tcp_start_queue()
1999 	nvme_tcp_setup_sock_ops(queue);  in nvme_tcp_start_queue()
2002 		nvme_tcp_set_queue_io_cpu(queue);  in nvme_tcp_start_queue()
2008 		set_bit(NVME_TCP_Q_LIVE, &queue->flags);  in nvme_tcp_start_queue()
2010 		if (test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))  in nvme_tcp_start_queue()
2011 			__nvme_tcp_stop_queue(queue);  in nvme_tcp_start_queue()
2579 static void nvme_tcp_set_sg_inline(struct nvme_tcp_queue *queue,  in nvme_tcp_set_sg_inline()  argument
2584 	sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff);  in nvme_tcp_set_sg_inline()
2603 	struct nvme_tcp_queue *queue = &ctrl->queues[0];  in nvme_tcp_submit_async_event()  local
2606 	u8 hdgst = nvme_tcp_hdgst_len(queue);  in nvme_tcp_submit_async_event()
2610 	if (queue->hdr_digest)  in nvme_tcp_submit_async_event()
2633 	struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;  in nvme_tcp_complete_timed_out()
2635 	nvme_tcp_stop_queue(ctrl, nvme_tcp_queue_id(req->queue));  in nvme_tcp_complete_timed_out()
2642 	struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;  in nvme_tcp_timeout()
2645 	int qid = nvme_tcp_queue_id(req->queue);  in nvme_tcp_timeout()
2678 static blk_status_t nvme_tcp_map_data(struct nvme_tcp_queue *queue,  in nvme_tcp_map_data()  argument
2691 		nvme_tcp_set_sg_inline(queue, c, req->data_len);  in nvme_tcp_map_data()
2703 	struct nvme_tcp_queue *queue = req->queue;  in nvme_tcp_setup_cmd_pdu()  local
2704 	u8 hdgst = nvme_tcp_hdgst_len(queue), ddgst = 0;  in nvme_tcp_setup_cmd_pdu()
2730 	if (queue->hdr_digest)  in nvme_tcp_setup_cmd_pdu()
2732 	if (queue->data_digest && req->pdu_len) {  in nvme_tcp_setup_cmd_pdu()
2734 		ddgst = nvme_tcp_ddgst_len(queue);  in nvme_tcp_setup_cmd_pdu()
2741 	ret = nvme_tcp_map_data(queue, rq);  in nvme_tcp_setup_cmd_pdu()
2744 		dev_err(queue->ctrl->ctrl.device,  in nvme_tcp_setup_cmd_pdu()
2754 	struct nvme_tcp_queue *queue = hctx->driver_data;  in nvme_tcp_commit_rqs()  local
2756 	if (!llist_empty(&queue->req_list))  in nvme_tcp_commit_rqs()
2757 		queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);  in nvme_tcp_commit_rqs()
2763 	struct nvme_ns *ns = hctx->queue->queuedata;  in nvme_tcp_queue_rq()
2764 	struct nvme_tcp_queue *queue = hctx->driver_data;  in nvme_tcp_queue_rq()  local
2767 	bool queue_ready = test_bit(NVME_TCP_Q_LIVE, &queue->flags);  in nvme_tcp_queue_rq()
2770 	if (!nvme_check_ready(&queue->ctrl->ctrl, rq, queue_ready))  in nvme_tcp_queue_rq()
2771 		return nvme_fail_nonready_command(&queue->ctrl->ctrl, rq);  in nvme_tcp_queue_rq()
2793 	struct nvme_tcp_queue *queue = hctx->driver_data;  in nvme_tcp_poll()  local
2794 	struct sock *sk = queue->sock->sk;  in nvme_tcp_poll()
2797 	if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags))  in nvme_tcp_poll()
2800 	set_bit(NVME_TCP_Q_POLLING, &queue->flags);  in nvme_tcp_poll()
2803 	ret = nvme_tcp_try_recv(queue);  in nvme_tcp_poll()
2804 	clear_bit(NVME_TCP_Q_POLLING, &queue->flags);  in nvme_tcp_poll()
2805 	return ret < 0 ? ret : queue->nr_cqe;  in nvme_tcp_poll()
2810 	struct nvme_tcp_queue *queue = &to_tcp_ctrl(ctrl)->queues[0];  in nvme_tcp_get_address()  local
2816 	if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags))  in nvme_tcp_get_address()
2819 	mutex_lock(&queue->queue_lock);  in nvme_tcp_get_address()
2821 	ret = kernel_getsockname(queue->sock, (struct sockaddr *)&src_addr);  in nvme_tcp_get_address()
2829 	mutex_unlock(&queue->queue_lock);  in nvme_tcp_get_address()