xref: /linux/drivers/nvme/target/tcp.c (revision 9b960d8cd6f712cb2c03e2bdd4d5ca058238037f)
1  // SPDX-License-Identifier: GPL-2.0
2  /*
3   * NVMe over Fabrics TCP target.
4   * Copyright (c) 2018 Lightbits Labs. All rights reserved.
5   */
6  #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7  #include <linux/module.h>
8  #include <linux/init.h>
9  #include <linux/slab.h>
10  #include <linux/err.h>
11  #include <linux/nvme-tcp.h>
12  #include <linux/nvme-keyring.h>
13  #include <net/sock.h>
14  #include <net/tcp.h>
15  #include <net/tls.h>
16  #include <net/tls_prot.h>
17  #include <net/handshake.h>
18  #include <linux/inet.h>
19  #include <linux/llist.h>
20  #include <crypto/hash.h>
21  #include <trace/events/sock.h>
22  
23  #include "nvmet.h"
24  
25  #define NVMET_TCP_DEF_INLINE_DATA_SIZE	(4 * PAGE_SIZE)
26  #define NVMET_TCP_MAXH2CDATA		0x400000 /* 16M arbitrary limit */
27  #define NVMET_TCP_BACKLOG 128
28  
param_store_val(const char * str,int * val,int min,int max)29  static int param_store_val(const char *str, int *val, int min, int max)
30  {
31  	int ret, new_val;
32  
33  	ret = kstrtoint(str, 10, &new_val);
34  	if (ret)
35  		return -EINVAL;
36  
37  	if (new_val < min || new_val > max)
38  		return -EINVAL;
39  
40  	*val = new_val;
41  	return 0;
42  }
43  
set_params(const char * str,const struct kernel_param * kp)44  static int set_params(const char *str, const struct kernel_param *kp)
45  {
46  	return param_store_val(str, kp->arg, 0, INT_MAX);
47  }
48  
49  static const struct kernel_param_ops set_param_ops = {
50  	.set	= set_params,
51  	.get	= param_get_int,
52  };
53  
54  /* Define the socket priority to use for connections were it is desirable
55   * that the NIC consider performing optimized packet processing or filtering.
56   * A non-zero value being sufficient to indicate general consideration of any
57   * possible optimization.  Making it a module param allows for alternative
58   * values that may be unique for some NIC implementations.
59   */
60  static int so_priority;
61  device_param_cb(so_priority, &set_param_ops, &so_priority, 0644);
62  MODULE_PARM_DESC(so_priority, "nvmet tcp socket optimize priority: Default 0");
63  
64  /* Define a time period (in usecs) that io_work() shall sample an activated
65   * queue before determining it to be idle.  This optional module behavior
66   * can enable NIC solutions that support socket optimized packet processing
67   * using advanced interrupt moderation techniques.
68   */
69  static int idle_poll_period_usecs;
70  device_param_cb(idle_poll_period_usecs, &set_param_ops,
71  		&idle_poll_period_usecs, 0644);
72  MODULE_PARM_DESC(idle_poll_period_usecs,
73  		"nvmet tcp io_work poll till idle time period in usecs: Default 0");
74  
75  #ifdef CONFIG_NVME_TARGET_TCP_TLS
76  /*
77   * TLS handshake timeout
78   */
79  static int tls_handshake_timeout = 10;
80  module_param(tls_handshake_timeout, int, 0644);
81  MODULE_PARM_DESC(tls_handshake_timeout,
82  		 "nvme TLS handshake timeout in seconds (default 10)");
83  #endif
84  
85  #define NVMET_TCP_RECV_BUDGET		8
86  #define NVMET_TCP_SEND_BUDGET		8
87  #define NVMET_TCP_IO_WORK_BUDGET	64
88  
89  enum nvmet_tcp_send_state {
90  	NVMET_TCP_SEND_DATA_PDU,
91  	NVMET_TCP_SEND_DATA,
92  	NVMET_TCP_SEND_R2T,
93  	NVMET_TCP_SEND_DDGST,
94  	NVMET_TCP_SEND_RESPONSE
95  };
96  
97  enum nvmet_tcp_recv_state {
98  	NVMET_TCP_RECV_PDU,
99  	NVMET_TCP_RECV_DATA,
100  	NVMET_TCP_RECV_DDGST,
101  	NVMET_TCP_RECV_ERR,
102  };
103  
104  enum {
105  	NVMET_TCP_F_INIT_FAILED = (1 << 0),
106  };
107  
108  struct nvmet_tcp_cmd {
109  	struct nvmet_tcp_queue		*queue;
110  	struct nvmet_req		req;
111  
112  	struct nvme_tcp_cmd_pdu		*cmd_pdu;
113  	struct nvme_tcp_rsp_pdu		*rsp_pdu;
114  	struct nvme_tcp_data_pdu	*data_pdu;
115  	struct nvme_tcp_r2t_pdu		*r2t_pdu;
116  
117  	u32				rbytes_done;
118  	u32				wbytes_done;
119  
120  	u32				pdu_len;
121  	u32				pdu_recv;
122  	int				sg_idx;
123  	char				recv_cbuf[CMSG_LEN(sizeof(char))];
124  	struct msghdr			recv_msg;
125  	struct bio_vec			*iov;
126  	u32				flags;
127  
128  	struct list_head		entry;
129  	struct llist_node		lentry;
130  
131  	/* send state */
132  	u32				offset;
133  	struct scatterlist		*cur_sg;
134  	enum nvmet_tcp_send_state	state;
135  
136  	__le32				exp_ddgst;
137  	__le32				recv_ddgst;
138  };
139  
140  enum nvmet_tcp_queue_state {
141  	NVMET_TCP_Q_CONNECTING,
142  	NVMET_TCP_Q_TLS_HANDSHAKE,
143  	NVMET_TCP_Q_LIVE,
144  	NVMET_TCP_Q_DISCONNECTING,
145  	NVMET_TCP_Q_FAILED,
146  };
147  
148  struct nvmet_tcp_queue {
149  	struct socket		*sock;
150  	struct nvmet_tcp_port	*port;
151  	struct work_struct	io_work;
152  	struct nvmet_cq		nvme_cq;
153  	struct nvmet_sq		nvme_sq;
154  	struct kref		kref;
155  
156  	/* send state */
157  	struct nvmet_tcp_cmd	*cmds;
158  	unsigned int		nr_cmds;
159  	struct list_head	free_list;
160  	struct llist_head	resp_list;
161  	struct list_head	resp_send_list;
162  	int			send_list_len;
163  	struct nvmet_tcp_cmd	*snd_cmd;
164  
165  	/* recv state */
166  	int			offset;
167  	int			left;
168  	enum nvmet_tcp_recv_state rcv_state;
169  	struct nvmet_tcp_cmd	*cmd;
170  	union nvme_tcp_pdu	pdu;
171  
172  	/* digest state */
173  	bool			hdr_digest;
174  	bool			data_digest;
175  	struct ahash_request	*snd_hash;
176  	struct ahash_request	*rcv_hash;
177  
178  	/* TLS state */
179  	key_serial_t		tls_pskid;
180  	struct delayed_work	tls_handshake_tmo_work;
181  
182  	unsigned long           poll_end;
183  
184  	spinlock_t		state_lock;
185  	enum nvmet_tcp_queue_state state;
186  
187  	struct sockaddr_storage	sockaddr;
188  	struct sockaddr_storage	sockaddr_peer;
189  	struct work_struct	release_work;
190  
191  	int			idx;
192  	struct list_head	queue_list;
193  
194  	struct nvmet_tcp_cmd	connect;
195  
196  	struct page_frag_cache	pf_cache;
197  
198  	void (*data_ready)(struct sock *);
199  	void (*state_change)(struct sock *);
200  	void (*write_space)(struct sock *);
201  };
202  
203  struct nvmet_tcp_port {
204  	struct socket		*sock;
205  	struct work_struct	accept_work;
206  	struct nvmet_port	*nport;
207  	struct sockaddr_storage addr;
208  	void (*data_ready)(struct sock *);
209  };
210  
211  static DEFINE_IDA(nvmet_tcp_queue_ida);
212  static LIST_HEAD(nvmet_tcp_queue_list);
213  static DEFINE_MUTEX(nvmet_tcp_queue_mutex);
214  
215  static struct workqueue_struct *nvmet_tcp_wq;
216  static const struct nvmet_fabrics_ops nvmet_tcp_ops;
217  static void nvmet_tcp_free_cmd(struct nvmet_tcp_cmd *c);
218  static void nvmet_tcp_free_cmd_buffers(struct nvmet_tcp_cmd *cmd);
219  
nvmet_tcp_cmd_tag(struct nvmet_tcp_queue * queue,struct nvmet_tcp_cmd * cmd)220  static inline u16 nvmet_tcp_cmd_tag(struct nvmet_tcp_queue *queue,
221  		struct nvmet_tcp_cmd *cmd)
222  {
223  	if (unlikely(!queue->nr_cmds)) {
224  		/* We didn't allocate cmds yet, send 0xffff */
225  		return USHRT_MAX;
226  	}
227  
228  	return cmd - queue->cmds;
229  }
230  
nvmet_tcp_has_data_in(struct nvmet_tcp_cmd * cmd)231  static inline bool nvmet_tcp_has_data_in(struct nvmet_tcp_cmd *cmd)
232  {
233  	return nvme_is_write(cmd->req.cmd) &&
234  		cmd->rbytes_done < cmd->req.transfer_len;
235  }
236  
nvmet_tcp_need_data_in(struct nvmet_tcp_cmd * cmd)237  static inline bool nvmet_tcp_need_data_in(struct nvmet_tcp_cmd *cmd)
238  {
239  	return nvmet_tcp_has_data_in(cmd) && !cmd->req.cqe->status;
240  }
241  
nvmet_tcp_need_data_out(struct nvmet_tcp_cmd * cmd)242  static inline bool nvmet_tcp_need_data_out(struct nvmet_tcp_cmd *cmd)
243  {
244  	return !nvme_is_write(cmd->req.cmd) &&
245  		cmd->req.transfer_len > 0 &&
246  		!cmd->req.cqe->status;
247  }
248  
nvmet_tcp_has_inline_data(struct nvmet_tcp_cmd * cmd)249  static inline bool nvmet_tcp_has_inline_data(struct nvmet_tcp_cmd *cmd)
250  {
251  	return nvme_is_write(cmd->req.cmd) && cmd->pdu_len &&
252  		!cmd->rbytes_done;
253  }
254  
255  static inline struct nvmet_tcp_cmd *
nvmet_tcp_get_cmd(struct nvmet_tcp_queue * queue)256  nvmet_tcp_get_cmd(struct nvmet_tcp_queue *queue)
257  {
258  	struct nvmet_tcp_cmd *cmd;
259  
260  	cmd = list_first_entry_or_null(&queue->free_list,
261  				struct nvmet_tcp_cmd, entry);
262  	if (!cmd)
263  		return NULL;
264  	list_del_init(&cmd->entry);
265  
266  	cmd->rbytes_done = cmd->wbytes_done = 0;
267  	cmd->pdu_len = 0;
268  	cmd->pdu_recv = 0;
269  	cmd->iov = NULL;
270  	cmd->flags = 0;
271  	return cmd;
272  }
273  
nvmet_tcp_put_cmd(struct nvmet_tcp_cmd * cmd)274  static inline void nvmet_tcp_put_cmd(struct nvmet_tcp_cmd *cmd)
275  {
276  	if (unlikely(cmd == &cmd->queue->connect))
277  		return;
278  
279  	list_add_tail(&cmd->entry, &cmd->queue->free_list);
280  }
281  
queue_cpu(struct nvmet_tcp_queue * queue)282  static inline int queue_cpu(struct nvmet_tcp_queue *queue)
283  {
284  	return queue->sock->sk->sk_incoming_cpu;
285  }
286  
nvmet_tcp_hdgst_len(struct nvmet_tcp_queue * queue)287  static inline u8 nvmet_tcp_hdgst_len(struct nvmet_tcp_queue *queue)
288  {
289  	return queue->hdr_digest ? NVME_TCP_DIGEST_LENGTH : 0;
290  }
291  
nvmet_tcp_ddgst_len(struct nvmet_tcp_queue * queue)292  static inline u8 nvmet_tcp_ddgst_len(struct nvmet_tcp_queue *queue)
293  {
294  	return queue->data_digest ? NVME_TCP_DIGEST_LENGTH : 0;
295  }
296  
nvmet_tcp_hdgst(struct ahash_request * hash,void * pdu,size_t len)297  static inline void nvmet_tcp_hdgst(struct ahash_request *hash,
298  		void *pdu, size_t len)
299  {
300  	struct scatterlist sg;
301  
302  	sg_init_one(&sg, pdu, len);
303  	ahash_request_set_crypt(hash, &sg, pdu + len, len);
304  	crypto_ahash_digest(hash);
305  }
306  
nvmet_tcp_verify_hdgst(struct nvmet_tcp_queue * queue,void * pdu,size_t len)307  static int nvmet_tcp_verify_hdgst(struct nvmet_tcp_queue *queue,
308  	void *pdu, size_t len)
309  {
310  	struct nvme_tcp_hdr *hdr = pdu;
311  	__le32 recv_digest;
312  	__le32 exp_digest;
313  
314  	if (unlikely(!(hdr->flags & NVME_TCP_F_HDGST))) {
315  		pr_err("queue %d: header digest enabled but no header digest\n",
316  			queue->idx);
317  		return -EPROTO;
318  	}
319  
320  	recv_digest = *(__le32 *)(pdu + hdr->hlen);
321  	nvmet_tcp_hdgst(queue->rcv_hash, pdu, len);
322  	exp_digest = *(__le32 *)(pdu + hdr->hlen);
323  	if (recv_digest != exp_digest) {
324  		pr_err("queue %d: header digest error: recv %#x expected %#x\n",
325  			queue->idx, le32_to_cpu(recv_digest),
326  			le32_to_cpu(exp_digest));
327  		return -EPROTO;
328  	}
329  
330  	return 0;
331  }
332  
nvmet_tcp_check_ddgst(struct nvmet_tcp_queue * queue,void * pdu)333  static int nvmet_tcp_check_ddgst(struct nvmet_tcp_queue *queue, void *pdu)
334  {
335  	struct nvme_tcp_hdr *hdr = pdu;
336  	u8 digest_len = nvmet_tcp_hdgst_len(queue);
337  	u32 len;
338  
339  	len = le32_to_cpu(hdr->plen) - hdr->hlen -
340  		(hdr->flags & NVME_TCP_F_HDGST ? digest_len : 0);
341  
342  	if (unlikely(len && !(hdr->flags & NVME_TCP_F_DDGST))) {
343  		pr_err("queue %d: data digest flag is cleared\n", queue->idx);
344  		return -EPROTO;
345  	}
346  
347  	return 0;
348  }
349  
350  /* If cmd buffers are NULL, no operation is performed */
nvmet_tcp_free_cmd_buffers(struct nvmet_tcp_cmd * cmd)351  static void nvmet_tcp_free_cmd_buffers(struct nvmet_tcp_cmd *cmd)
352  {
353  	kfree(cmd->iov);
354  	sgl_free(cmd->req.sg);
355  	cmd->iov = NULL;
356  	cmd->req.sg = NULL;
357  }
358  
nvmet_tcp_build_pdu_iovec(struct nvmet_tcp_cmd * cmd)359  static void nvmet_tcp_build_pdu_iovec(struct nvmet_tcp_cmd *cmd)
360  {
361  	struct bio_vec *iov = cmd->iov;
362  	struct scatterlist *sg;
363  	u32 length, offset, sg_offset;
364  	int nr_pages;
365  
366  	length = cmd->pdu_len;
367  	nr_pages = DIV_ROUND_UP(length, PAGE_SIZE);
368  	offset = cmd->rbytes_done;
369  	cmd->sg_idx = offset / PAGE_SIZE;
370  	sg_offset = offset % PAGE_SIZE;
371  	sg = &cmd->req.sg[cmd->sg_idx];
372  
373  	while (length) {
374  		u32 iov_len = min_t(u32, length, sg->length - sg_offset);
375  
376  		bvec_set_page(iov, sg_page(sg), iov_len,
377  				sg->offset + sg_offset);
378  
379  		length -= iov_len;
380  		sg = sg_next(sg);
381  		iov++;
382  		sg_offset = 0;
383  	}
384  
385  	iov_iter_bvec(&cmd->recv_msg.msg_iter, ITER_DEST, cmd->iov,
386  		      nr_pages, cmd->pdu_len);
387  }
388  
nvmet_tcp_fatal_error(struct nvmet_tcp_queue * queue)389  static void nvmet_tcp_fatal_error(struct nvmet_tcp_queue *queue)
390  {
391  	queue->rcv_state = NVMET_TCP_RECV_ERR;
392  	if (queue->nvme_sq.ctrl)
393  		nvmet_ctrl_fatal_error(queue->nvme_sq.ctrl);
394  	else
395  		kernel_sock_shutdown(queue->sock, SHUT_RDWR);
396  }
397  
nvmet_tcp_socket_error(struct nvmet_tcp_queue * queue,int status)398  static void nvmet_tcp_socket_error(struct nvmet_tcp_queue *queue, int status)
399  {
400  	queue->rcv_state = NVMET_TCP_RECV_ERR;
401  	if (status == -EPIPE || status == -ECONNRESET)
402  		kernel_sock_shutdown(queue->sock, SHUT_RDWR);
403  	else
404  		nvmet_tcp_fatal_error(queue);
405  }
406  
nvmet_tcp_map_data(struct nvmet_tcp_cmd * cmd)407  static int nvmet_tcp_map_data(struct nvmet_tcp_cmd *cmd)
408  {
409  	struct nvme_sgl_desc *sgl = &cmd->req.cmd->common.dptr.sgl;
410  	u32 len = le32_to_cpu(sgl->length);
411  
412  	if (!len)
413  		return 0;
414  
415  	if (sgl->type == ((NVME_SGL_FMT_DATA_DESC << 4) |
416  			  NVME_SGL_FMT_OFFSET)) {
417  		if (!nvme_is_write(cmd->req.cmd))
418  			return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
419  
420  		if (len > cmd->req.port->inline_data_size)
421  			return NVME_SC_SGL_INVALID_OFFSET | NVME_STATUS_DNR;
422  		cmd->pdu_len = len;
423  	}
424  	cmd->req.transfer_len += len;
425  
426  	cmd->req.sg = sgl_alloc(len, GFP_KERNEL, &cmd->req.sg_cnt);
427  	if (!cmd->req.sg)
428  		return NVME_SC_INTERNAL;
429  	cmd->cur_sg = cmd->req.sg;
430  
431  	if (nvmet_tcp_has_data_in(cmd)) {
432  		cmd->iov = kmalloc_array(cmd->req.sg_cnt,
433  				sizeof(*cmd->iov), GFP_KERNEL);
434  		if (!cmd->iov)
435  			goto err;
436  	}
437  
438  	return 0;
439  err:
440  	nvmet_tcp_free_cmd_buffers(cmd);
441  	return NVME_SC_INTERNAL;
442  }
443  
nvmet_tcp_calc_ddgst(struct ahash_request * hash,struct nvmet_tcp_cmd * cmd)444  static void nvmet_tcp_calc_ddgst(struct ahash_request *hash,
445  		struct nvmet_tcp_cmd *cmd)
446  {
447  	ahash_request_set_crypt(hash, cmd->req.sg,
448  		(void *)&cmd->exp_ddgst, cmd->req.transfer_len);
449  	crypto_ahash_digest(hash);
450  }
451  
nvmet_setup_c2h_data_pdu(struct nvmet_tcp_cmd * cmd)452  static void nvmet_setup_c2h_data_pdu(struct nvmet_tcp_cmd *cmd)
453  {
454  	struct nvme_tcp_data_pdu *pdu = cmd->data_pdu;
455  	struct nvmet_tcp_queue *queue = cmd->queue;
456  	u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue);
457  	u8 ddgst = nvmet_tcp_ddgst_len(cmd->queue);
458  
459  	cmd->offset = 0;
460  	cmd->state = NVMET_TCP_SEND_DATA_PDU;
461  
462  	pdu->hdr.type = nvme_tcp_c2h_data;
463  	pdu->hdr.flags = NVME_TCP_F_DATA_LAST | (queue->nvme_sq.sqhd_disabled ?
464  						NVME_TCP_F_DATA_SUCCESS : 0);
465  	pdu->hdr.hlen = sizeof(*pdu);
466  	pdu->hdr.pdo = pdu->hdr.hlen + hdgst;
467  	pdu->hdr.plen =
468  		cpu_to_le32(pdu->hdr.hlen + hdgst +
469  				cmd->req.transfer_len + ddgst);
470  	pdu->command_id = cmd->req.cqe->command_id;
471  	pdu->data_length = cpu_to_le32(cmd->req.transfer_len);
472  	pdu->data_offset = cpu_to_le32(cmd->wbytes_done);
473  
474  	if (queue->data_digest) {
475  		pdu->hdr.flags |= NVME_TCP_F_DDGST;
476  		nvmet_tcp_calc_ddgst(queue->snd_hash, cmd);
477  	}
478  
479  	if (cmd->queue->hdr_digest) {
480  		pdu->hdr.flags |= NVME_TCP_F_HDGST;
481  		nvmet_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
482  	}
483  }
484  
nvmet_setup_r2t_pdu(struct nvmet_tcp_cmd * cmd)485  static void nvmet_setup_r2t_pdu(struct nvmet_tcp_cmd *cmd)
486  {
487  	struct nvme_tcp_r2t_pdu *pdu = cmd->r2t_pdu;
488  	struct nvmet_tcp_queue *queue = cmd->queue;
489  	u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue);
490  
491  	cmd->offset = 0;
492  	cmd->state = NVMET_TCP_SEND_R2T;
493  
494  	pdu->hdr.type = nvme_tcp_r2t;
495  	pdu->hdr.flags = 0;
496  	pdu->hdr.hlen = sizeof(*pdu);
497  	pdu->hdr.pdo = 0;
498  	pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);
499  
500  	pdu->command_id = cmd->req.cmd->common.command_id;
501  	pdu->ttag = nvmet_tcp_cmd_tag(cmd->queue, cmd);
502  	pdu->r2t_length = cpu_to_le32(cmd->req.transfer_len - cmd->rbytes_done);
503  	pdu->r2t_offset = cpu_to_le32(cmd->rbytes_done);
504  	if (cmd->queue->hdr_digest) {
505  		pdu->hdr.flags |= NVME_TCP_F_HDGST;
506  		nvmet_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
507  	}
508  }
509  
nvmet_setup_response_pdu(struct nvmet_tcp_cmd * cmd)510  static void nvmet_setup_response_pdu(struct nvmet_tcp_cmd *cmd)
511  {
512  	struct nvme_tcp_rsp_pdu *pdu = cmd->rsp_pdu;
513  	struct nvmet_tcp_queue *queue = cmd->queue;
514  	u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue);
515  
516  	cmd->offset = 0;
517  	cmd->state = NVMET_TCP_SEND_RESPONSE;
518  
519  	pdu->hdr.type = nvme_tcp_rsp;
520  	pdu->hdr.flags = 0;
521  	pdu->hdr.hlen = sizeof(*pdu);
522  	pdu->hdr.pdo = 0;
523  	pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);
524  	if (cmd->queue->hdr_digest) {
525  		pdu->hdr.flags |= NVME_TCP_F_HDGST;
526  		nvmet_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
527  	}
528  }
529  
nvmet_tcp_process_resp_list(struct nvmet_tcp_queue * queue)530  static void nvmet_tcp_process_resp_list(struct nvmet_tcp_queue *queue)
531  {
532  	struct llist_node *node;
533  	struct nvmet_tcp_cmd *cmd;
534  
535  	for (node = llist_del_all(&queue->resp_list); node; node = node->next) {
536  		cmd = llist_entry(node, struct nvmet_tcp_cmd, lentry);
537  		list_add(&cmd->entry, &queue->resp_send_list);
538  		queue->send_list_len++;
539  	}
540  }
541  
nvmet_tcp_fetch_cmd(struct nvmet_tcp_queue * queue)542  static struct nvmet_tcp_cmd *nvmet_tcp_fetch_cmd(struct nvmet_tcp_queue *queue)
543  {
544  	queue->snd_cmd = list_first_entry_or_null(&queue->resp_send_list,
545  				struct nvmet_tcp_cmd, entry);
546  	if (!queue->snd_cmd) {
547  		nvmet_tcp_process_resp_list(queue);
548  		queue->snd_cmd =
549  			list_first_entry_or_null(&queue->resp_send_list,
550  					struct nvmet_tcp_cmd, entry);
551  		if (unlikely(!queue->snd_cmd))
552  			return NULL;
553  	}
554  
555  	list_del_init(&queue->snd_cmd->entry);
556  	queue->send_list_len--;
557  
558  	if (nvmet_tcp_need_data_out(queue->snd_cmd))
559  		nvmet_setup_c2h_data_pdu(queue->snd_cmd);
560  	else if (nvmet_tcp_need_data_in(queue->snd_cmd))
561  		nvmet_setup_r2t_pdu(queue->snd_cmd);
562  	else
563  		nvmet_setup_response_pdu(queue->snd_cmd);
564  
565  	return queue->snd_cmd;
566  }
567  
nvmet_tcp_queue_response(struct nvmet_req * req)568  static void nvmet_tcp_queue_response(struct nvmet_req *req)
569  {
570  	struct nvmet_tcp_cmd *cmd =
571  		container_of(req, struct nvmet_tcp_cmd, req);
572  	struct nvmet_tcp_queue	*queue = cmd->queue;
573  	enum nvmet_tcp_recv_state queue_state;
574  	struct nvmet_tcp_cmd *queue_cmd;
575  	struct nvme_sgl_desc *sgl;
576  	u32 len;
577  
578  	/* Pairs with store_release in nvmet_prepare_receive_pdu() */
579  	queue_state = smp_load_acquire(&queue->rcv_state);
580  	queue_cmd = READ_ONCE(queue->cmd);
581  
582  	if (unlikely(cmd == queue_cmd)) {
583  		sgl = &cmd->req.cmd->common.dptr.sgl;
584  		len = le32_to_cpu(sgl->length);
585  
586  		/*
587  		 * Wait for inline data before processing the response.
588  		 * Avoid using helpers, this might happen before
589  		 * nvmet_req_init is completed.
590  		 */
591  		if (queue_state == NVMET_TCP_RECV_PDU &&
592  		    len && len <= cmd->req.port->inline_data_size &&
593  		    nvme_is_write(cmd->req.cmd))
594  			return;
595  	}
596  
597  	llist_add(&cmd->lentry, &queue->resp_list);
598  	queue_work_on(queue_cpu(queue), nvmet_tcp_wq, &cmd->queue->io_work);
599  }
600  
nvmet_tcp_execute_request(struct nvmet_tcp_cmd * cmd)601  static void nvmet_tcp_execute_request(struct nvmet_tcp_cmd *cmd)
602  {
603  	if (unlikely(cmd->flags & NVMET_TCP_F_INIT_FAILED))
604  		nvmet_tcp_queue_response(&cmd->req);
605  	else
606  		cmd->req.execute(&cmd->req);
607  }
608  
nvmet_try_send_data_pdu(struct nvmet_tcp_cmd * cmd)609  static int nvmet_try_send_data_pdu(struct nvmet_tcp_cmd *cmd)
610  {
611  	struct msghdr msg = {
612  		.msg_flags = MSG_DONTWAIT | MSG_MORE | MSG_SPLICE_PAGES,
613  	};
614  	struct bio_vec bvec;
615  	u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue);
616  	int left = sizeof(*cmd->data_pdu) - cmd->offset + hdgst;
617  	int ret;
618  
619  	bvec_set_virt(&bvec, (void *)cmd->data_pdu + cmd->offset, left);
620  	iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, left);
621  	ret = sock_sendmsg(cmd->queue->sock, &msg);
622  	if (ret <= 0)
623  		return ret;
624  
625  	cmd->offset += ret;
626  	left -= ret;
627  
628  	if (left)
629  		return -EAGAIN;
630  
631  	cmd->state = NVMET_TCP_SEND_DATA;
632  	cmd->offset  = 0;
633  	return 1;
634  }
635  
nvmet_try_send_data(struct nvmet_tcp_cmd * cmd,bool last_in_batch)636  static int nvmet_try_send_data(struct nvmet_tcp_cmd *cmd, bool last_in_batch)
637  {
638  	struct nvmet_tcp_queue *queue = cmd->queue;
639  	int ret;
640  
641  	while (cmd->cur_sg) {
642  		struct msghdr msg = {
643  			.msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES,
644  		};
645  		struct page *page = sg_page(cmd->cur_sg);
646  		struct bio_vec bvec;
647  		u32 left = cmd->cur_sg->length - cmd->offset;
648  
649  		if ((!last_in_batch && cmd->queue->send_list_len) ||
650  		    cmd->wbytes_done + left < cmd->req.transfer_len ||
651  		    queue->data_digest || !queue->nvme_sq.sqhd_disabled)
652  			msg.msg_flags |= MSG_MORE;
653  
654  		bvec_set_page(&bvec, page, left, cmd->offset);
655  		iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, left);
656  		ret = sock_sendmsg(cmd->queue->sock, &msg);
657  		if (ret <= 0)
658  			return ret;
659  
660  		cmd->offset += ret;
661  		cmd->wbytes_done += ret;
662  
663  		/* Done with sg?*/
664  		if (cmd->offset == cmd->cur_sg->length) {
665  			cmd->cur_sg = sg_next(cmd->cur_sg);
666  			cmd->offset = 0;
667  		}
668  	}
669  
670  	if (queue->data_digest) {
671  		cmd->state = NVMET_TCP_SEND_DDGST;
672  		cmd->offset = 0;
673  	} else {
674  		if (queue->nvme_sq.sqhd_disabled) {
675  			cmd->queue->snd_cmd = NULL;
676  			nvmet_tcp_put_cmd(cmd);
677  		} else {
678  			nvmet_setup_response_pdu(cmd);
679  		}
680  	}
681  
682  	if (queue->nvme_sq.sqhd_disabled)
683  		nvmet_tcp_free_cmd_buffers(cmd);
684  
685  	return 1;
686  
687  }
688  
nvmet_try_send_response(struct nvmet_tcp_cmd * cmd,bool last_in_batch)689  static int nvmet_try_send_response(struct nvmet_tcp_cmd *cmd,
690  		bool last_in_batch)
691  {
692  	struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES, };
693  	struct bio_vec bvec;
694  	u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue);
695  	int left = sizeof(*cmd->rsp_pdu) - cmd->offset + hdgst;
696  	int ret;
697  
698  	if (!last_in_batch && cmd->queue->send_list_len)
699  		msg.msg_flags |= MSG_MORE;
700  	else
701  		msg.msg_flags |= MSG_EOR;
702  
703  	bvec_set_virt(&bvec, (void *)cmd->rsp_pdu + cmd->offset, left);
704  	iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, left);
705  	ret = sock_sendmsg(cmd->queue->sock, &msg);
706  	if (ret <= 0)
707  		return ret;
708  	cmd->offset += ret;
709  	left -= ret;
710  
711  	if (left)
712  		return -EAGAIN;
713  
714  	nvmet_tcp_free_cmd_buffers(cmd);
715  	cmd->queue->snd_cmd = NULL;
716  	nvmet_tcp_put_cmd(cmd);
717  	return 1;
718  }
719  
nvmet_try_send_r2t(struct nvmet_tcp_cmd * cmd,bool last_in_batch)720  static int nvmet_try_send_r2t(struct nvmet_tcp_cmd *cmd, bool last_in_batch)
721  {
722  	struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES, };
723  	struct bio_vec bvec;
724  	u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue);
725  	int left = sizeof(*cmd->r2t_pdu) - cmd->offset + hdgst;
726  	int ret;
727  
728  	if (!last_in_batch && cmd->queue->send_list_len)
729  		msg.msg_flags |= MSG_MORE;
730  	else
731  		msg.msg_flags |= MSG_EOR;
732  
733  	bvec_set_virt(&bvec, (void *)cmd->r2t_pdu + cmd->offset, left);
734  	iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, left);
735  	ret = sock_sendmsg(cmd->queue->sock, &msg);
736  	if (ret <= 0)
737  		return ret;
738  	cmd->offset += ret;
739  	left -= ret;
740  
741  	if (left)
742  		return -EAGAIN;
743  
744  	cmd->queue->snd_cmd = NULL;
745  	return 1;
746  }
747  
nvmet_try_send_ddgst(struct nvmet_tcp_cmd * cmd,bool last_in_batch)748  static int nvmet_try_send_ddgst(struct nvmet_tcp_cmd *cmd, bool last_in_batch)
749  {
750  	struct nvmet_tcp_queue *queue = cmd->queue;
751  	int left = NVME_TCP_DIGEST_LENGTH - cmd->offset;
752  	struct msghdr msg = { .msg_flags = MSG_DONTWAIT };
753  	struct kvec iov = {
754  		.iov_base = (u8 *)&cmd->exp_ddgst + cmd->offset,
755  		.iov_len = left
756  	};
757  	int ret;
758  
759  	if (!last_in_batch && cmd->queue->send_list_len)
760  		msg.msg_flags |= MSG_MORE;
761  	else
762  		msg.msg_flags |= MSG_EOR;
763  
764  	ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
765  	if (unlikely(ret <= 0))
766  		return ret;
767  
768  	cmd->offset += ret;
769  	left -= ret;
770  
771  	if (left)
772  		return -EAGAIN;
773  
774  	if (queue->nvme_sq.sqhd_disabled) {
775  		cmd->queue->snd_cmd = NULL;
776  		nvmet_tcp_put_cmd(cmd);
777  	} else {
778  		nvmet_setup_response_pdu(cmd);
779  	}
780  	return 1;
781  }
782  
nvmet_tcp_try_send_one(struct nvmet_tcp_queue * queue,bool last_in_batch)783  static int nvmet_tcp_try_send_one(struct nvmet_tcp_queue *queue,
784  		bool last_in_batch)
785  {
786  	struct nvmet_tcp_cmd *cmd = queue->snd_cmd;
787  	int ret = 0;
788  
789  	if (!cmd || queue->state == NVMET_TCP_Q_DISCONNECTING) {
790  		cmd = nvmet_tcp_fetch_cmd(queue);
791  		if (unlikely(!cmd))
792  			return 0;
793  	}
794  
795  	if (cmd->state == NVMET_TCP_SEND_DATA_PDU) {
796  		ret = nvmet_try_send_data_pdu(cmd);
797  		if (ret <= 0)
798  			goto done_send;
799  	}
800  
801  	if (cmd->state == NVMET_TCP_SEND_DATA) {
802  		ret = nvmet_try_send_data(cmd, last_in_batch);
803  		if (ret <= 0)
804  			goto done_send;
805  	}
806  
807  	if (cmd->state == NVMET_TCP_SEND_DDGST) {
808  		ret = nvmet_try_send_ddgst(cmd, last_in_batch);
809  		if (ret <= 0)
810  			goto done_send;
811  	}
812  
813  	if (cmd->state == NVMET_TCP_SEND_R2T) {
814  		ret = nvmet_try_send_r2t(cmd, last_in_batch);
815  		if (ret <= 0)
816  			goto done_send;
817  	}
818  
819  	if (cmd->state == NVMET_TCP_SEND_RESPONSE)
820  		ret = nvmet_try_send_response(cmd, last_in_batch);
821  
822  done_send:
823  	if (ret < 0) {
824  		if (ret == -EAGAIN)
825  			return 0;
826  		return ret;
827  	}
828  
829  	return 1;
830  }
831  
nvmet_tcp_try_send(struct nvmet_tcp_queue * queue,int budget,int * sends)832  static int nvmet_tcp_try_send(struct nvmet_tcp_queue *queue,
833  		int budget, int *sends)
834  {
835  	int i, ret = 0;
836  
837  	for (i = 0; i < budget; i++) {
838  		ret = nvmet_tcp_try_send_one(queue, i == budget - 1);
839  		if (unlikely(ret < 0)) {
840  			nvmet_tcp_socket_error(queue, ret);
841  			goto done;
842  		} else if (ret == 0) {
843  			break;
844  		}
845  		(*sends)++;
846  	}
847  done:
848  	return ret;
849  }
850  
nvmet_prepare_receive_pdu(struct nvmet_tcp_queue * queue)851  static void nvmet_prepare_receive_pdu(struct nvmet_tcp_queue *queue)
852  {
853  	queue->offset = 0;
854  	queue->left = sizeof(struct nvme_tcp_hdr);
855  	WRITE_ONCE(queue->cmd, NULL);
856  	/* Ensure rcv_state is visible only after queue->cmd is set */
857  	smp_store_release(&queue->rcv_state, NVMET_TCP_RECV_PDU);
858  }
859  
nvmet_tcp_free_crypto(struct nvmet_tcp_queue * queue)860  static void nvmet_tcp_free_crypto(struct nvmet_tcp_queue *queue)
861  {
862  	struct crypto_ahash *tfm = crypto_ahash_reqtfm(queue->rcv_hash);
863  
864  	ahash_request_free(queue->rcv_hash);
865  	ahash_request_free(queue->snd_hash);
866  	crypto_free_ahash(tfm);
867  }
868  
nvmet_tcp_alloc_crypto(struct nvmet_tcp_queue * queue)869  static int nvmet_tcp_alloc_crypto(struct nvmet_tcp_queue *queue)
870  {
871  	struct crypto_ahash *tfm;
872  
873  	tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC);
874  	if (IS_ERR(tfm))
875  		return PTR_ERR(tfm);
876  
877  	queue->snd_hash = ahash_request_alloc(tfm, GFP_KERNEL);
878  	if (!queue->snd_hash)
879  		goto free_tfm;
880  	ahash_request_set_callback(queue->snd_hash, 0, NULL, NULL);
881  
882  	queue->rcv_hash = ahash_request_alloc(tfm, GFP_KERNEL);
883  	if (!queue->rcv_hash)
884  		goto free_snd_hash;
885  	ahash_request_set_callback(queue->rcv_hash, 0, NULL, NULL);
886  
887  	return 0;
888  free_snd_hash:
889  	ahash_request_free(queue->snd_hash);
890  free_tfm:
891  	crypto_free_ahash(tfm);
892  	return -ENOMEM;
893  }
894  
895  
nvmet_tcp_handle_icreq(struct nvmet_tcp_queue * queue)896  static int nvmet_tcp_handle_icreq(struct nvmet_tcp_queue *queue)
897  {
898  	struct nvme_tcp_icreq_pdu *icreq = &queue->pdu.icreq;
899  	struct nvme_tcp_icresp_pdu *icresp = &queue->pdu.icresp;
900  	struct msghdr msg = {};
901  	struct kvec iov;
902  	int ret;
903  
904  	if (le32_to_cpu(icreq->hdr.plen) != sizeof(struct nvme_tcp_icreq_pdu)) {
905  		pr_err("bad nvme-tcp pdu length (%d)\n",
906  			le32_to_cpu(icreq->hdr.plen));
907  		nvmet_tcp_fatal_error(queue);
908  		return -EPROTO;
909  	}
910  
911  	if (icreq->pfv != NVME_TCP_PFV_1_0) {
912  		pr_err("queue %d: bad pfv %d\n", queue->idx, icreq->pfv);
913  		return -EPROTO;
914  	}
915  
916  	if (icreq->hpda != 0) {
917  		pr_err("queue %d: unsupported hpda %d\n", queue->idx,
918  			icreq->hpda);
919  		return -EPROTO;
920  	}
921  
922  	queue->hdr_digest = !!(icreq->digest & NVME_TCP_HDR_DIGEST_ENABLE);
923  	queue->data_digest = !!(icreq->digest & NVME_TCP_DATA_DIGEST_ENABLE);
924  	if (queue->hdr_digest || queue->data_digest) {
925  		ret = nvmet_tcp_alloc_crypto(queue);
926  		if (ret)
927  			return ret;
928  	}
929  
930  	memset(icresp, 0, sizeof(*icresp));
931  	icresp->hdr.type = nvme_tcp_icresp;
932  	icresp->hdr.hlen = sizeof(*icresp);
933  	icresp->hdr.pdo = 0;
934  	icresp->hdr.plen = cpu_to_le32(icresp->hdr.hlen);
935  	icresp->pfv = cpu_to_le16(NVME_TCP_PFV_1_0);
936  	icresp->maxdata = cpu_to_le32(NVMET_TCP_MAXH2CDATA);
937  	icresp->cpda = 0;
938  	if (queue->hdr_digest)
939  		icresp->digest |= NVME_TCP_HDR_DIGEST_ENABLE;
940  	if (queue->data_digest)
941  		icresp->digest |= NVME_TCP_DATA_DIGEST_ENABLE;
942  
943  	iov.iov_base = icresp;
944  	iov.iov_len = sizeof(*icresp);
945  	ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
946  	if (ret < 0) {
947  		queue->state = NVMET_TCP_Q_FAILED;
948  		return ret; /* queue removal will cleanup */
949  	}
950  
951  	queue->state = NVMET_TCP_Q_LIVE;
952  	nvmet_prepare_receive_pdu(queue);
953  	return 0;
954  }
955  
nvmet_tcp_handle_req_failure(struct nvmet_tcp_queue * queue,struct nvmet_tcp_cmd * cmd,struct nvmet_req * req)956  static void nvmet_tcp_handle_req_failure(struct nvmet_tcp_queue *queue,
957  		struct nvmet_tcp_cmd *cmd, struct nvmet_req *req)
958  {
959  	size_t data_len = le32_to_cpu(req->cmd->common.dptr.sgl.length);
960  	int ret;
961  
962  	/*
963  	 * This command has not been processed yet, hence we are trying to
964  	 * figure out if there is still pending data left to receive. If
965  	 * we don't, we can simply prepare for the next pdu and bail out,
966  	 * otherwise we will need to prepare a buffer and receive the
967  	 * stale data before continuing forward.
968  	 */
969  	if (!nvme_is_write(cmd->req.cmd) || !data_len ||
970  	    data_len > cmd->req.port->inline_data_size) {
971  		nvmet_prepare_receive_pdu(queue);
972  		return;
973  	}
974  
975  	ret = nvmet_tcp_map_data(cmd);
976  	if (unlikely(ret)) {
977  		pr_err("queue %d: failed to map data\n", queue->idx);
978  		nvmet_tcp_fatal_error(queue);
979  		return;
980  	}
981  
982  	queue->rcv_state = NVMET_TCP_RECV_DATA;
983  	nvmet_tcp_build_pdu_iovec(cmd);
984  	cmd->flags |= NVMET_TCP_F_INIT_FAILED;
985  }
986  
nvmet_tcp_handle_h2c_data_pdu(struct nvmet_tcp_queue * queue)987  static int nvmet_tcp_handle_h2c_data_pdu(struct nvmet_tcp_queue *queue)
988  {
989  	struct nvme_tcp_data_pdu *data = &queue->pdu.data;
990  	struct nvmet_tcp_cmd *cmd;
991  	unsigned int exp_data_len;
992  
993  	if (likely(queue->nr_cmds)) {
994  		if (unlikely(data->ttag >= queue->nr_cmds)) {
995  			pr_err("queue %d: received out of bound ttag %u, nr_cmds %u\n",
996  				queue->idx, data->ttag, queue->nr_cmds);
997  			goto err_proto;
998  		}
999  		cmd = &queue->cmds[data->ttag];
1000  	} else {
1001  		cmd = &queue->connect;
1002  	}
1003  
1004  	if (le32_to_cpu(data->data_offset) != cmd->rbytes_done) {
1005  		pr_err("ttag %u unexpected data offset %u (expected %u)\n",
1006  			data->ttag, le32_to_cpu(data->data_offset),
1007  			cmd->rbytes_done);
1008  		goto err_proto;
1009  	}
1010  
1011  	exp_data_len = le32_to_cpu(data->hdr.plen) -
1012  			nvmet_tcp_hdgst_len(queue) -
1013  			nvmet_tcp_ddgst_len(queue) -
1014  			sizeof(*data);
1015  
1016  	cmd->pdu_len = le32_to_cpu(data->data_length);
1017  	if (unlikely(cmd->pdu_len != exp_data_len ||
1018  		     cmd->pdu_len == 0 ||
1019  		     cmd->pdu_len > NVMET_TCP_MAXH2CDATA)) {
1020  		pr_err("H2CData PDU len %u is invalid\n", cmd->pdu_len);
1021  		goto err_proto;
1022  	}
1023  	cmd->pdu_recv = 0;
1024  	nvmet_tcp_build_pdu_iovec(cmd);
1025  	queue->cmd = cmd;
1026  	queue->rcv_state = NVMET_TCP_RECV_DATA;
1027  
1028  	return 0;
1029  
1030  err_proto:
1031  	/* FIXME: use proper transport errors */
1032  	nvmet_tcp_fatal_error(queue);
1033  	return -EPROTO;
1034  }
1035  
nvmet_tcp_done_recv_pdu(struct nvmet_tcp_queue * queue)1036  static int nvmet_tcp_done_recv_pdu(struct nvmet_tcp_queue *queue)
1037  {
1038  	struct nvme_tcp_hdr *hdr = &queue->pdu.cmd.hdr;
1039  	struct nvme_command *nvme_cmd = &queue->pdu.cmd.cmd;
1040  	struct nvmet_req *req;
1041  	int ret;
1042  
1043  	if (unlikely(queue->state == NVMET_TCP_Q_CONNECTING)) {
1044  		if (hdr->type != nvme_tcp_icreq) {
1045  			pr_err("unexpected pdu type (%d) before icreq\n",
1046  				hdr->type);
1047  			nvmet_tcp_fatal_error(queue);
1048  			return -EPROTO;
1049  		}
1050  		return nvmet_tcp_handle_icreq(queue);
1051  	}
1052  
1053  	if (unlikely(hdr->type == nvme_tcp_icreq)) {
1054  		pr_err("queue %d: received icreq pdu in state %d\n",
1055  			queue->idx, queue->state);
1056  		nvmet_tcp_fatal_error(queue);
1057  		return -EPROTO;
1058  	}
1059  
1060  	if (hdr->type == nvme_tcp_h2c_data) {
1061  		ret = nvmet_tcp_handle_h2c_data_pdu(queue);
1062  		if (unlikely(ret))
1063  			return ret;
1064  		return 0;
1065  	}
1066  
1067  	queue->cmd = nvmet_tcp_get_cmd(queue);
1068  	if (unlikely(!queue->cmd)) {
1069  		/* This should never happen */
1070  		pr_err("queue %d: out of commands (%d) send_list_len: %d, opcode: %d",
1071  			queue->idx, queue->nr_cmds, queue->send_list_len,
1072  			nvme_cmd->common.opcode);
1073  		nvmet_tcp_fatal_error(queue);
1074  		return -ENOMEM;
1075  	}
1076  
1077  	req = &queue->cmd->req;
1078  	memcpy(req->cmd, nvme_cmd, sizeof(*nvme_cmd));
1079  
1080  	if (unlikely(!nvmet_req_init(req, &queue->nvme_cq,
1081  			&queue->nvme_sq, &nvmet_tcp_ops))) {
1082  		pr_err("failed cmd %p id %d opcode %d, data_len: %d, status: %04x\n",
1083  			req->cmd, req->cmd->common.command_id,
1084  			req->cmd->common.opcode,
1085  			le32_to_cpu(req->cmd->common.dptr.sgl.length),
1086  			le16_to_cpu(req->cqe->status));
1087  
1088  		nvmet_tcp_handle_req_failure(queue, queue->cmd, req);
1089  		return 0;
1090  	}
1091  
1092  	ret = nvmet_tcp_map_data(queue->cmd);
1093  	if (unlikely(ret)) {
1094  		pr_err("queue %d: failed to map data\n", queue->idx);
1095  		if (nvmet_tcp_has_inline_data(queue->cmd))
1096  			nvmet_tcp_fatal_error(queue);
1097  		else
1098  			nvmet_req_complete(req, ret);
1099  		ret = -EAGAIN;
1100  		goto out;
1101  	}
1102  
1103  	if (nvmet_tcp_need_data_in(queue->cmd)) {
1104  		if (nvmet_tcp_has_inline_data(queue->cmd)) {
1105  			queue->rcv_state = NVMET_TCP_RECV_DATA;
1106  			nvmet_tcp_build_pdu_iovec(queue->cmd);
1107  			return 0;
1108  		}
1109  		/* send back R2T */
1110  		nvmet_tcp_queue_response(&queue->cmd->req);
1111  		goto out;
1112  	}
1113  
1114  	queue->cmd->req.execute(&queue->cmd->req);
1115  out:
1116  	nvmet_prepare_receive_pdu(queue);
1117  	return ret;
1118  }
1119  
1120  static const u8 nvme_tcp_pdu_sizes[] = {
1121  	[nvme_tcp_icreq]	= sizeof(struct nvme_tcp_icreq_pdu),
1122  	[nvme_tcp_cmd]		= sizeof(struct nvme_tcp_cmd_pdu),
1123  	[nvme_tcp_h2c_data]	= sizeof(struct nvme_tcp_data_pdu),
1124  };
1125  
nvmet_tcp_pdu_size(u8 type)1126  static inline u8 nvmet_tcp_pdu_size(u8 type)
1127  {
1128  	size_t idx = type;
1129  
1130  	return (idx < ARRAY_SIZE(nvme_tcp_pdu_sizes) &&
1131  		nvme_tcp_pdu_sizes[idx]) ?
1132  			nvme_tcp_pdu_sizes[idx] : 0;
1133  }
1134  
nvmet_tcp_pdu_valid(u8 type)1135  static inline bool nvmet_tcp_pdu_valid(u8 type)
1136  {
1137  	switch (type) {
1138  	case nvme_tcp_icreq:
1139  	case nvme_tcp_cmd:
1140  	case nvme_tcp_h2c_data:
1141  		/* fallthru */
1142  		return true;
1143  	}
1144  
1145  	return false;
1146  }
1147  
nvmet_tcp_tls_record_ok(struct nvmet_tcp_queue * queue,struct msghdr * msg,char * cbuf)1148  static int nvmet_tcp_tls_record_ok(struct nvmet_tcp_queue *queue,
1149  		struct msghdr *msg, char *cbuf)
1150  {
1151  	struct cmsghdr *cmsg = (struct cmsghdr *)cbuf;
1152  	u8 ctype, level, description;
1153  	int ret = 0;
1154  
1155  	ctype = tls_get_record_type(queue->sock->sk, cmsg);
1156  	switch (ctype) {
1157  	case 0:
1158  		break;
1159  	case TLS_RECORD_TYPE_DATA:
1160  		break;
1161  	case TLS_RECORD_TYPE_ALERT:
1162  		tls_alert_recv(queue->sock->sk, msg, &level, &description);
1163  		if (level == TLS_ALERT_LEVEL_FATAL) {
1164  			pr_err("queue %d: TLS Alert desc %u\n",
1165  			       queue->idx, description);
1166  			ret = -ENOTCONN;
1167  		} else {
1168  			pr_warn("queue %d: TLS Alert desc %u\n",
1169  			       queue->idx, description);
1170  			ret = -EAGAIN;
1171  		}
1172  		break;
1173  	default:
1174  		/* discard this record type */
1175  		pr_err("queue %d: TLS record %d unhandled\n",
1176  		       queue->idx, ctype);
1177  		ret = -EAGAIN;
1178  		break;
1179  	}
1180  	return ret;
1181  }
1182  
nvmet_tcp_try_recv_pdu(struct nvmet_tcp_queue * queue)1183  static int nvmet_tcp_try_recv_pdu(struct nvmet_tcp_queue *queue)
1184  {
1185  	struct nvme_tcp_hdr *hdr = &queue->pdu.cmd.hdr;
1186  	int len, ret;
1187  	struct kvec iov;
1188  	char cbuf[CMSG_LEN(sizeof(char))] = {};
1189  	struct msghdr msg = { .msg_flags = MSG_DONTWAIT };
1190  
1191  recv:
1192  	iov.iov_base = (void *)&queue->pdu + queue->offset;
1193  	iov.iov_len = queue->left;
1194  	if (queue->tls_pskid) {
1195  		msg.msg_control = cbuf;
1196  		msg.msg_controllen = sizeof(cbuf);
1197  	}
1198  	len = kernel_recvmsg(queue->sock, &msg, &iov, 1,
1199  			iov.iov_len, msg.msg_flags);
1200  	if (unlikely(len < 0))
1201  		return len;
1202  	if (queue->tls_pskid) {
1203  		ret = nvmet_tcp_tls_record_ok(queue, &msg, cbuf);
1204  		if (ret < 0)
1205  			return ret;
1206  	}
1207  
1208  	queue->offset += len;
1209  	queue->left -= len;
1210  	if (queue->left)
1211  		return -EAGAIN;
1212  
1213  	if (queue->offset == sizeof(struct nvme_tcp_hdr)) {
1214  		u8 hdgst = nvmet_tcp_hdgst_len(queue);
1215  
1216  		if (unlikely(!nvmet_tcp_pdu_valid(hdr->type))) {
1217  			pr_err("unexpected pdu type %d\n", hdr->type);
1218  			nvmet_tcp_fatal_error(queue);
1219  			return -EIO;
1220  		}
1221  
1222  		if (unlikely(hdr->hlen != nvmet_tcp_pdu_size(hdr->type))) {
1223  			pr_err("pdu %d bad hlen %d\n", hdr->type, hdr->hlen);
1224  			return -EIO;
1225  		}
1226  
1227  		queue->left = hdr->hlen - queue->offset + hdgst;
1228  		goto recv;
1229  	}
1230  
1231  	if (queue->hdr_digest &&
1232  	    nvmet_tcp_verify_hdgst(queue, &queue->pdu, hdr->hlen)) {
1233  		nvmet_tcp_fatal_error(queue); /* fatal */
1234  		return -EPROTO;
1235  	}
1236  
1237  	if (queue->data_digest &&
1238  	    nvmet_tcp_check_ddgst(queue, &queue->pdu)) {
1239  		nvmet_tcp_fatal_error(queue); /* fatal */
1240  		return -EPROTO;
1241  	}
1242  
1243  	return nvmet_tcp_done_recv_pdu(queue);
1244  }
1245  
nvmet_tcp_prep_recv_ddgst(struct nvmet_tcp_cmd * cmd)1246  static void nvmet_tcp_prep_recv_ddgst(struct nvmet_tcp_cmd *cmd)
1247  {
1248  	struct nvmet_tcp_queue *queue = cmd->queue;
1249  
1250  	nvmet_tcp_calc_ddgst(queue->rcv_hash, cmd);
1251  	queue->offset = 0;
1252  	queue->left = NVME_TCP_DIGEST_LENGTH;
1253  	queue->rcv_state = NVMET_TCP_RECV_DDGST;
1254  }
1255  
nvmet_tcp_try_recv_data(struct nvmet_tcp_queue * queue)1256  static int nvmet_tcp_try_recv_data(struct nvmet_tcp_queue *queue)
1257  {
1258  	struct nvmet_tcp_cmd  *cmd = queue->cmd;
1259  	int len, ret;
1260  
1261  	while (msg_data_left(&cmd->recv_msg)) {
1262  		len = sock_recvmsg(cmd->queue->sock, &cmd->recv_msg,
1263  			cmd->recv_msg.msg_flags);
1264  		if (len <= 0)
1265  			return len;
1266  		if (queue->tls_pskid) {
1267  			ret = nvmet_tcp_tls_record_ok(cmd->queue,
1268  					&cmd->recv_msg, cmd->recv_cbuf);
1269  			if (ret < 0)
1270  				return ret;
1271  		}
1272  
1273  		cmd->pdu_recv += len;
1274  		cmd->rbytes_done += len;
1275  	}
1276  
1277  	if (queue->data_digest) {
1278  		nvmet_tcp_prep_recv_ddgst(cmd);
1279  		return 0;
1280  	}
1281  
1282  	if (cmd->rbytes_done == cmd->req.transfer_len)
1283  		nvmet_tcp_execute_request(cmd);
1284  
1285  	nvmet_prepare_receive_pdu(queue);
1286  	return 0;
1287  }
1288  
nvmet_tcp_try_recv_ddgst(struct nvmet_tcp_queue * queue)1289  static int nvmet_tcp_try_recv_ddgst(struct nvmet_tcp_queue *queue)
1290  {
1291  	struct nvmet_tcp_cmd *cmd = queue->cmd;
1292  	int ret, len;
1293  	char cbuf[CMSG_LEN(sizeof(char))] = {};
1294  	struct msghdr msg = { .msg_flags = MSG_DONTWAIT };
1295  	struct kvec iov = {
1296  		.iov_base = (void *)&cmd->recv_ddgst + queue->offset,
1297  		.iov_len = queue->left
1298  	};
1299  
1300  	if (queue->tls_pskid) {
1301  		msg.msg_control = cbuf;
1302  		msg.msg_controllen = sizeof(cbuf);
1303  	}
1304  	len = kernel_recvmsg(queue->sock, &msg, &iov, 1,
1305  			iov.iov_len, msg.msg_flags);
1306  	if (unlikely(len < 0))
1307  		return len;
1308  	if (queue->tls_pskid) {
1309  		ret = nvmet_tcp_tls_record_ok(queue, &msg, cbuf);
1310  		if (ret < 0)
1311  			return ret;
1312  	}
1313  
1314  	queue->offset += len;
1315  	queue->left -= len;
1316  	if (queue->left)
1317  		return -EAGAIN;
1318  
1319  	if (queue->data_digest && cmd->exp_ddgst != cmd->recv_ddgst) {
1320  		pr_err("queue %d: cmd %d pdu (%d) data digest error: recv %#x expected %#x\n",
1321  			queue->idx, cmd->req.cmd->common.command_id,
1322  			queue->pdu.cmd.hdr.type, le32_to_cpu(cmd->recv_ddgst),
1323  			le32_to_cpu(cmd->exp_ddgst));
1324  		nvmet_req_uninit(&cmd->req);
1325  		nvmet_tcp_free_cmd_buffers(cmd);
1326  		nvmet_tcp_fatal_error(queue);
1327  		ret = -EPROTO;
1328  		goto out;
1329  	}
1330  
1331  	if (cmd->rbytes_done == cmd->req.transfer_len)
1332  		nvmet_tcp_execute_request(cmd);
1333  
1334  	ret = 0;
1335  out:
1336  	nvmet_prepare_receive_pdu(queue);
1337  	return ret;
1338  }
1339  
nvmet_tcp_try_recv_one(struct nvmet_tcp_queue * queue)1340  static int nvmet_tcp_try_recv_one(struct nvmet_tcp_queue *queue)
1341  {
1342  	int result = 0;
1343  
1344  	if (unlikely(queue->rcv_state == NVMET_TCP_RECV_ERR))
1345  		return 0;
1346  
1347  	if (queue->rcv_state == NVMET_TCP_RECV_PDU) {
1348  		result = nvmet_tcp_try_recv_pdu(queue);
1349  		if (result != 0)
1350  			goto done_recv;
1351  	}
1352  
1353  	if (queue->rcv_state == NVMET_TCP_RECV_DATA) {
1354  		result = nvmet_tcp_try_recv_data(queue);
1355  		if (result != 0)
1356  			goto done_recv;
1357  	}
1358  
1359  	if (queue->rcv_state == NVMET_TCP_RECV_DDGST) {
1360  		result = nvmet_tcp_try_recv_ddgst(queue);
1361  		if (result != 0)
1362  			goto done_recv;
1363  	}
1364  
1365  done_recv:
1366  	if (result < 0) {
1367  		if (result == -EAGAIN)
1368  			return 0;
1369  		return result;
1370  	}
1371  	return 1;
1372  }
1373  
nvmet_tcp_try_recv(struct nvmet_tcp_queue * queue,int budget,int * recvs)1374  static int nvmet_tcp_try_recv(struct nvmet_tcp_queue *queue,
1375  		int budget, int *recvs)
1376  {
1377  	int i, ret = 0;
1378  
1379  	for (i = 0; i < budget; i++) {
1380  		ret = nvmet_tcp_try_recv_one(queue);
1381  		if (unlikely(ret < 0)) {
1382  			nvmet_tcp_socket_error(queue, ret);
1383  			goto done;
1384  		} else if (ret == 0) {
1385  			break;
1386  		}
1387  		(*recvs)++;
1388  	}
1389  done:
1390  	return ret;
1391  }
1392  
nvmet_tcp_release_queue(struct kref * kref)1393  static void nvmet_tcp_release_queue(struct kref *kref)
1394  {
1395  	struct nvmet_tcp_queue *queue =
1396  		container_of(kref, struct nvmet_tcp_queue, kref);
1397  
1398  	WARN_ON(queue->state != NVMET_TCP_Q_DISCONNECTING);
1399  	queue_work(nvmet_wq, &queue->release_work);
1400  }
1401  
nvmet_tcp_schedule_release_queue(struct nvmet_tcp_queue * queue)1402  static void nvmet_tcp_schedule_release_queue(struct nvmet_tcp_queue *queue)
1403  {
1404  	spin_lock_bh(&queue->state_lock);
1405  	if (queue->state == NVMET_TCP_Q_TLS_HANDSHAKE) {
1406  		/* Socket closed during handshake */
1407  		tls_handshake_cancel(queue->sock->sk);
1408  	}
1409  	if (queue->state != NVMET_TCP_Q_DISCONNECTING) {
1410  		queue->state = NVMET_TCP_Q_DISCONNECTING;
1411  		kref_put(&queue->kref, nvmet_tcp_release_queue);
1412  	}
1413  	spin_unlock_bh(&queue->state_lock);
1414  }
1415  
nvmet_tcp_arm_queue_deadline(struct nvmet_tcp_queue * queue)1416  static inline void nvmet_tcp_arm_queue_deadline(struct nvmet_tcp_queue *queue)
1417  {
1418  	queue->poll_end = jiffies + usecs_to_jiffies(idle_poll_period_usecs);
1419  }
1420  
nvmet_tcp_check_queue_deadline(struct nvmet_tcp_queue * queue,int ops)1421  static bool nvmet_tcp_check_queue_deadline(struct nvmet_tcp_queue *queue,
1422  		int ops)
1423  {
1424  	if (!idle_poll_period_usecs)
1425  		return false;
1426  
1427  	if (ops)
1428  		nvmet_tcp_arm_queue_deadline(queue);
1429  
1430  	return !time_after(jiffies, queue->poll_end);
1431  }
1432  
nvmet_tcp_io_work(struct work_struct * w)1433  static void nvmet_tcp_io_work(struct work_struct *w)
1434  {
1435  	struct nvmet_tcp_queue *queue =
1436  		container_of(w, struct nvmet_tcp_queue, io_work);
1437  	bool pending;
1438  	int ret, ops = 0;
1439  
1440  	do {
1441  		pending = false;
1442  
1443  		ret = nvmet_tcp_try_recv(queue, NVMET_TCP_RECV_BUDGET, &ops);
1444  		if (ret > 0)
1445  			pending = true;
1446  		else if (ret < 0)
1447  			return;
1448  
1449  		ret = nvmet_tcp_try_send(queue, NVMET_TCP_SEND_BUDGET, &ops);
1450  		if (ret > 0)
1451  			pending = true;
1452  		else if (ret < 0)
1453  			return;
1454  
1455  	} while (pending && ops < NVMET_TCP_IO_WORK_BUDGET);
1456  
1457  	/*
1458  	 * Requeue the worker if idle deadline period is in progress or any
1459  	 * ops activity was recorded during the do-while loop above.
1460  	 */
1461  	if (nvmet_tcp_check_queue_deadline(queue, ops) || pending)
1462  		queue_work_on(queue_cpu(queue), nvmet_tcp_wq, &queue->io_work);
1463  }
1464  
nvmet_tcp_alloc_cmd(struct nvmet_tcp_queue * queue,struct nvmet_tcp_cmd * c)1465  static int nvmet_tcp_alloc_cmd(struct nvmet_tcp_queue *queue,
1466  		struct nvmet_tcp_cmd *c)
1467  {
1468  	u8 hdgst = nvmet_tcp_hdgst_len(queue);
1469  
1470  	c->queue = queue;
1471  	c->req.port = queue->port->nport;
1472  
1473  	c->cmd_pdu = page_frag_alloc(&queue->pf_cache,
1474  			sizeof(*c->cmd_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO);
1475  	if (!c->cmd_pdu)
1476  		return -ENOMEM;
1477  	c->req.cmd = &c->cmd_pdu->cmd;
1478  
1479  	c->rsp_pdu = page_frag_alloc(&queue->pf_cache,
1480  			sizeof(*c->rsp_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO);
1481  	if (!c->rsp_pdu)
1482  		goto out_free_cmd;
1483  	c->req.cqe = &c->rsp_pdu->cqe;
1484  
1485  	c->data_pdu = page_frag_alloc(&queue->pf_cache,
1486  			sizeof(*c->data_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO);
1487  	if (!c->data_pdu)
1488  		goto out_free_rsp;
1489  
1490  	c->r2t_pdu = page_frag_alloc(&queue->pf_cache,
1491  			sizeof(*c->r2t_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO);
1492  	if (!c->r2t_pdu)
1493  		goto out_free_data;
1494  
1495  	if (queue->state == NVMET_TCP_Q_TLS_HANDSHAKE) {
1496  		c->recv_msg.msg_control = c->recv_cbuf;
1497  		c->recv_msg.msg_controllen = sizeof(c->recv_cbuf);
1498  	}
1499  	c->recv_msg.msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL;
1500  
1501  	list_add_tail(&c->entry, &queue->free_list);
1502  
1503  	return 0;
1504  out_free_data:
1505  	page_frag_free(c->data_pdu);
1506  out_free_rsp:
1507  	page_frag_free(c->rsp_pdu);
1508  out_free_cmd:
1509  	page_frag_free(c->cmd_pdu);
1510  	return -ENOMEM;
1511  }
1512  
nvmet_tcp_free_cmd(struct nvmet_tcp_cmd * c)1513  static void nvmet_tcp_free_cmd(struct nvmet_tcp_cmd *c)
1514  {
1515  	page_frag_free(c->r2t_pdu);
1516  	page_frag_free(c->data_pdu);
1517  	page_frag_free(c->rsp_pdu);
1518  	page_frag_free(c->cmd_pdu);
1519  }
1520  
nvmet_tcp_alloc_cmds(struct nvmet_tcp_queue * queue)1521  static int nvmet_tcp_alloc_cmds(struct nvmet_tcp_queue *queue)
1522  {
1523  	struct nvmet_tcp_cmd *cmds;
1524  	int i, ret = -EINVAL, nr_cmds = queue->nr_cmds;
1525  
1526  	cmds = kcalloc(nr_cmds, sizeof(struct nvmet_tcp_cmd), GFP_KERNEL);
1527  	if (!cmds)
1528  		goto out;
1529  
1530  	for (i = 0; i < nr_cmds; i++) {
1531  		ret = nvmet_tcp_alloc_cmd(queue, cmds + i);
1532  		if (ret)
1533  			goto out_free;
1534  	}
1535  
1536  	queue->cmds = cmds;
1537  
1538  	return 0;
1539  out_free:
1540  	while (--i >= 0)
1541  		nvmet_tcp_free_cmd(cmds + i);
1542  	kfree(cmds);
1543  out:
1544  	return ret;
1545  }
1546  
nvmet_tcp_free_cmds(struct nvmet_tcp_queue * queue)1547  static void nvmet_tcp_free_cmds(struct nvmet_tcp_queue *queue)
1548  {
1549  	struct nvmet_tcp_cmd *cmds = queue->cmds;
1550  	int i;
1551  
1552  	for (i = 0; i < queue->nr_cmds; i++)
1553  		nvmet_tcp_free_cmd(cmds + i);
1554  
1555  	nvmet_tcp_free_cmd(&queue->connect);
1556  	kfree(cmds);
1557  }
1558  
nvmet_tcp_restore_socket_callbacks(struct nvmet_tcp_queue * queue)1559  static void nvmet_tcp_restore_socket_callbacks(struct nvmet_tcp_queue *queue)
1560  {
1561  	struct socket *sock = queue->sock;
1562  
1563  	write_lock_bh(&sock->sk->sk_callback_lock);
1564  	sock->sk->sk_data_ready =  queue->data_ready;
1565  	sock->sk->sk_state_change = queue->state_change;
1566  	sock->sk->sk_write_space = queue->write_space;
1567  	sock->sk->sk_user_data = NULL;
1568  	write_unlock_bh(&sock->sk->sk_callback_lock);
1569  }
1570  
nvmet_tcp_uninit_data_in_cmds(struct nvmet_tcp_queue * queue)1571  static void nvmet_tcp_uninit_data_in_cmds(struct nvmet_tcp_queue *queue)
1572  {
1573  	struct nvmet_tcp_cmd *cmd = queue->cmds;
1574  	int i;
1575  
1576  	for (i = 0; i < queue->nr_cmds; i++, cmd++) {
1577  		if (nvmet_tcp_need_data_in(cmd))
1578  			nvmet_req_uninit(&cmd->req);
1579  	}
1580  
1581  	if (!queue->nr_cmds && nvmet_tcp_need_data_in(&queue->connect)) {
1582  		/* failed in connect */
1583  		nvmet_req_uninit(&queue->connect.req);
1584  	}
1585  }
1586  
nvmet_tcp_free_cmd_data_in_buffers(struct nvmet_tcp_queue * queue)1587  static void nvmet_tcp_free_cmd_data_in_buffers(struct nvmet_tcp_queue *queue)
1588  {
1589  	struct nvmet_tcp_cmd *cmd = queue->cmds;
1590  	int i;
1591  
1592  	for (i = 0; i < queue->nr_cmds; i++, cmd++)
1593  		nvmet_tcp_free_cmd_buffers(cmd);
1594  	nvmet_tcp_free_cmd_buffers(&queue->connect);
1595  }
1596  
nvmet_tcp_release_queue_work(struct work_struct * w)1597  static void nvmet_tcp_release_queue_work(struct work_struct *w)
1598  {
1599  	struct nvmet_tcp_queue *queue =
1600  		container_of(w, struct nvmet_tcp_queue, release_work);
1601  
1602  	mutex_lock(&nvmet_tcp_queue_mutex);
1603  	list_del_init(&queue->queue_list);
1604  	mutex_unlock(&nvmet_tcp_queue_mutex);
1605  
1606  	nvmet_tcp_restore_socket_callbacks(queue);
1607  	cancel_delayed_work_sync(&queue->tls_handshake_tmo_work);
1608  	cancel_work_sync(&queue->io_work);
1609  	/* stop accepting incoming data */
1610  	queue->rcv_state = NVMET_TCP_RECV_ERR;
1611  
1612  	nvmet_sq_put_tls_key(&queue->nvme_sq);
1613  	nvmet_tcp_uninit_data_in_cmds(queue);
1614  	nvmet_sq_destroy(&queue->nvme_sq);
1615  	cancel_work_sync(&queue->io_work);
1616  	nvmet_tcp_free_cmd_data_in_buffers(queue);
1617  	/* ->sock will be released by fput() */
1618  	fput(queue->sock->file);
1619  	nvmet_tcp_free_cmds(queue);
1620  	if (queue->hdr_digest || queue->data_digest)
1621  		nvmet_tcp_free_crypto(queue);
1622  	ida_free(&nvmet_tcp_queue_ida, queue->idx);
1623  	page_frag_cache_drain(&queue->pf_cache);
1624  	kfree(queue);
1625  }
1626  
nvmet_tcp_data_ready(struct sock * sk)1627  static void nvmet_tcp_data_ready(struct sock *sk)
1628  {
1629  	struct nvmet_tcp_queue *queue;
1630  
1631  	trace_sk_data_ready(sk);
1632  
1633  	read_lock_bh(&sk->sk_callback_lock);
1634  	queue = sk->sk_user_data;
1635  	if (likely(queue)) {
1636  		if (queue->data_ready)
1637  			queue->data_ready(sk);
1638  		if (queue->state != NVMET_TCP_Q_TLS_HANDSHAKE)
1639  			queue_work_on(queue_cpu(queue), nvmet_tcp_wq,
1640  				      &queue->io_work);
1641  	}
1642  	read_unlock_bh(&sk->sk_callback_lock);
1643  }
1644  
nvmet_tcp_write_space(struct sock * sk)1645  static void nvmet_tcp_write_space(struct sock *sk)
1646  {
1647  	struct nvmet_tcp_queue *queue;
1648  
1649  	read_lock_bh(&sk->sk_callback_lock);
1650  	queue = sk->sk_user_data;
1651  	if (unlikely(!queue))
1652  		goto out;
1653  
1654  	if (unlikely(queue->state == NVMET_TCP_Q_CONNECTING)) {
1655  		queue->write_space(sk);
1656  		goto out;
1657  	}
1658  
1659  	if (sk_stream_is_writeable(sk)) {
1660  		clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1661  		queue_work_on(queue_cpu(queue), nvmet_tcp_wq, &queue->io_work);
1662  	}
1663  out:
1664  	read_unlock_bh(&sk->sk_callback_lock);
1665  }
1666  
nvmet_tcp_state_change(struct sock * sk)1667  static void nvmet_tcp_state_change(struct sock *sk)
1668  {
1669  	struct nvmet_tcp_queue *queue;
1670  
1671  	read_lock_bh(&sk->sk_callback_lock);
1672  	queue = sk->sk_user_data;
1673  	if (!queue)
1674  		goto done;
1675  
1676  	switch (sk->sk_state) {
1677  	case TCP_FIN_WAIT2:
1678  	case TCP_LAST_ACK:
1679  		break;
1680  	case TCP_FIN_WAIT1:
1681  	case TCP_CLOSE_WAIT:
1682  	case TCP_CLOSE:
1683  		/* FALLTHRU */
1684  		nvmet_tcp_schedule_release_queue(queue);
1685  		break;
1686  	default:
1687  		pr_warn("queue %d unhandled state %d\n",
1688  			queue->idx, sk->sk_state);
1689  	}
1690  done:
1691  	read_unlock_bh(&sk->sk_callback_lock);
1692  }
1693  
nvmet_tcp_set_queue_sock(struct nvmet_tcp_queue * queue)1694  static int nvmet_tcp_set_queue_sock(struct nvmet_tcp_queue *queue)
1695  {
1696  	struct socket *sock = queue->sock;
1697  	struct inet_sock *inet = inet_sk(sock->sk);
1698  	int ret;
1699  
1700  	ret = kernel_getsockname(sock,
1701  		(struct sockaddr *)&queue->sockaddr);
1702  	if (ret < 0)
1703  		return ret;
1704  
1705  	ret = kernel_getpeername(sock,
1706  		(struct sockaddr *)&queue->sockaddr_peer);
1707  	if (ret < 0)
1708  		return ret;
1709  
1710  	/*
1711  	 * Cleanup whatever is sitting in the TCP transmit queue on socket
1712  	 * close. This is done to prevent stale data from being sent should
1713  	 * the network connection be restored before TCP times out.
1714  	 */
1715  	sock_no_linger(sock->sk);
1716  
1717  	if (so_priority > 0)
1718  		sock_set_priority(sock->sk, so_priority);
1719  
1720  	/* Set socket type of service */
1721  	if (inet->rcv_tos > 0)
1722  		ip_sock_set_tos(sock->sk, inet->rcv_tos);
1723  
1724  	ret = 0;
1725  	write_lock_bh(&sock->sk->sk_callback_lock);
1726  	if (sock->sk->sk_state != TCP_ESTABLISHED) {
1727  		/*
1728  		 * If the socket is already closing, don't even start
1729  		 * consuming it
1730  		 */
1731  		ret = -ENOTCONN;
1732  	} else {
1733  		sock->sk->sk_user_data = queue;
1734  		queue->data_ready = sock->sk->sk_data_ready;
1735  		sock->sk->sk_data_ready = nvmet_tcp_data_ready;
1736  		queue->state_change = sock->sk->sk_state_change;
1737  		sock->sk->sk_state_change = nvmet_tcp_state_change;
1738  		queue->write_space = sock->sk->sk_write_space;
1739  		sock->sk->sk_write_space = nvmet_tcp_write_space;
1740  		if (idle_poll_period_usecs)
1741  			nvmet_tcp_arm_queue_deadline(queue);
1742  		queue_work_on(queue_cpu(queue), nvmet_tcp_wq, &queue->io_work);
1743  	}
1744  	write_unlock_bh(&sock->sk->sk_callback_lock);
1745  
1746  	return ret;
1747  }
1748  
1749  #ifdef CONFIG_NVME_TARGET_TCP_TLS
nvmet_tcp_try_peek_pdu(struct nvmet_tcp_queue * queue)1750  static int nvmet_tcp_try_peek_pdu(struct nvmet_tcp_queue *queue)
1751  {
1752  	struct nvme_tcp_hdr *hdr = &queue->pdu.cmd.hdr;
1753  	int len, ret;
1754  	struct kvec iov = {
1755  		.iov_base = (u8 *)&queue->pdu + queue->offset,
1756  		.iov_len = sizeof(struct nvme_tcp_hdr),
1757  	};
1758  	char cbuf[CMSG_LEN(sizeof(char))] = {};
1759  	struct msghdr msg = {
1760  		.msg_control = cbuf,
1761  		.msg_controllen = sizeof(cbuf),
1762  		.msg_flags = MSG_PEEK,
1763  	};
1764  
1765  	if (nvmet_port_secure_channel_required(queue->port->nport))
1766  		return 0;
1767  
1768  	len = kernel_recvmsg(queue->sock, &msg, &iov, 1,
1769  			iov.iov_len, msg.msg_flags);
1770  	if (unlikely(len < 0)) {
1771  		pr_debug("queue %d: peek error %d\n",
1772  			 queue->idx, len);
1773  		return len;
1774  	}
1775  
1776  	ret = nvmet_tcp_tls_record_ok(queue, &msg, cbuf);
1777  	if (ret < 0)
1778  		return ret;
1779  
1780  	if (len < sizeof(struct nvme_tcp_hdr)) {
1781  		pr_debug("queue %d: short read, %d bytes missing\n",
1782  			 queue->idx, (int)iov.iov_len - len);
1783  		return -EAGAIN;
1784  	}
1785  	pr_debug("queue %d: hdr type %d hlen %d plen %d size %d\n",
1786  		 queue->idx, hdr->type, hdr->hlen, hdr->plen,
1787  		 (int)sizeof(struct nvme_tcp_icreq_pdu));
1788  	if (hdr->type == nvme_tcp_icreq &&
1789  	    hdr->hlen == sizeof(struct nvme_tcp_icreq_pdu) &&
1790  	    hdr->plen == cpu_to_le32(sizeof(struct nvme_tcp_icreq_pdu))) {
1791  		pr_debug("queue %d: icreq detected\n",
1792  			 queue->idx);
1793  		return len;
1794  	}
1795  	return 0;
1796  }
1797  
nvmet_tcp_tls_key_lookup(struct nvmet_tcp_queue * queue,key_serial_t peerid)1798  static int nvmet_tcp_tls_key_lookup(struct nvmet_tcp_queue *queue,
1799  				    key_serial_t peerid)
1800  {
1801  	struct key *tls_key = nvme_tls_key_lookup(peerid);
1802  	int status = 0;
1803  
1804  	if (IS_ERR(tls_key)) {
1805  		pr_warn("%s: queue %d failed to lookup key %x\n",
1806  			__func__, queue->idx, peerid);
1807  		spin_lock_bh(&queue->state_lock);
1808  		queue->state = NVMET_TCP_Q_FAILED;
1809  		spin_unlock_bh(&queue->state_lock);
1810  		status = PTR_ERR(tls_key);
1811  	} else {
1812  		pr_debug("%s: queue %d using TLS PSK %x\n",
1813  			 __func__, queue->idx, peerid);
1814  		queue->nvme_sq.tls_key = tls_key;
1815  	}
1816  	return status;
1817  }
1818  
nvmet_tcp_tls_handshake_done(void * data,int status,key_serial_t peerid)1819  static void nvmet_tcp_tls_handshake_done(void *data, int status,
1820  					 key_serial_t peerid)
1821  {
1822  	struct nvmet_tcp_queue *queue = data;
1823  
1824  	pr_debug("queue %d: TLS handshake done, key %x, status %d\n",
1825  		 queue->idx, peerid, status);
1826  	spin_lock_bh(&queue->state_lock);
1827  	if (WARN_ON(queue->state != NVMET_TCP_Q_TLS_HANDSHAKE)) {
1828  		spin_unlock_bh(&queue->state_lock);
1829  		return;
1830  	}
1831  	if (!status) {
1832  		queue->tls_pskid = peerid;
1833  		queue->state = NVMET_TCP_Q_CONNECTING;
1834  	} else
1835  		queue->state = NVMET_TCP_Q_FAILED;
1836  	spin_unlock_bh(&queue->state_lock);
1837  
1838  	cancel_delayed_work_sync(&queue->tls_handshake_tmo_work);
1839  
1840  	if (!status)
1841  		status = nvmet_tcp_tls_key_lookup(queue, peerid);
1842  
1843  	if (status)
1844  		nvmet_tcp_schedule_release_queue(queue);
1845  	else
1846  		nvmet_tcp_set_queue_sock(queue);
1847  	kref_put(&queue->kref, nvmet_tcp_release_queue);
1848  }
1849  
nvmet_tcp_tls_handshake_timeout(struct work_struct * w)1850  static void nvmet_tcp_tls_handshake_timeout(struct work_struct *w)
1851  {
1852  	struct nvmet_tcp_queue *queue = container_of(to_delayed_work(w),
1853  			struct nvmet_tcp_queue, tls_handshake_tmo_work);
1854  
1855  	pr_warn("queue %d: TLS handshake timeout\n", queue->idx);
1856  	/*
1857  	 * If tls_handshake_cancel() fails we've lost the race with
1858  	 * nvmet_tcp_tls_handshake_done() */
1859  	if (!tls_handshake_cancel(queue->sock->sk))
1860  		return;
1861  	spin_lock_bh(&queue->state_lock);
1862  	if (WARN_ON(queue->state != NVMET_TCP_Q_TLS_HANDSHAKE)) {
1863  		spin_unlock_bh(&queue->state_lock);
1864  		return;
1865  	}
1866  	queue->state = NVMET_TCP_Q_FAILED;
1867  	spin_unlock_bh(&queue->state_lock);
1868  	nvmet_tcp_schedule_release_queue(queue);
1869  	kref_put(&queue->kref, nvmet_tcp_release_queue);
1870  }
1871  
nvmet_tcp_tls_handshake(struct nvmet_tcp_queue * queue)1872  static int nvmet_tcp_tls_handshake(struct nvmet_tcp_queue *queue)
1873  {
1874  	int ret = -EOPNOTSUPP;
1875  	struct tls_handshake_args args;
1876  
1877  	if (queue->state != NVMET_TCP_Q_TLS_HANDSHAKE) {
1878  		pr_warn("cannot start TLS in state %d\n", queue->state);
1879  		return -EINVAL;
1880  	}
1881  
1882  	kref_get(&queue->kref);
1883  	pr_debug("queue %d: TLS ServerHello\n", queue->idx);
1884  	memset(&args, 0, sizeof(args));
1885  	args.ta_sock = queue->sock;
1886  	args.ta_done = nvmet_tcp_tls_handshake_done;
1887  	args.ta_data = queue;
1888  	args.ta_keyring = key_serial(queue->port->nport->keyring);
1889  	args.ta_timeout_ms = tls_handshake_timeout * 1000;
1890  
1891  	ret = tls_server_hello_psk(&args, GFP_KERNEL);
1892  	if (ret) {
1893  		kref_put(&queue->kref, nvmet_tcp_release_queue);
1894  		pr_err("failed to start TLS, err=%d\n", ret);
1895  	} else {
1896  		queue_delayed_work(nvmet_wq, &queue->tls_handshake_tmo_work,
1897  				   tls_handshake_timeout * HZ);
1898  	}
1899  	return ret;
1900  }
1901  #else
nvmet_tcp_tls_handshake_timeout(struct work_struct * w)1902  static void nvmet_tcp_tls_handshake_timeout(struct work_struct *w) {}
1903  #endif
1904  
nvmet_tcp_alloc_queue(struct nvmet_tcp_port * port,struct socket * newsock)1905  static void nvmet_tcp_alloc_queue(struct nvmet_tcp_port *port,
1906  		struct socket *newsock)
1907  {
1908  	struct nvmet_tcp_queue *queue;
1909  	struct file *sock_file = NULL;
1910  	int ret;
1911  
1912  	queue = kzalloc(sizeof(*queue), GFP_KERNEL);
1913  	if (!queue) {
1914  		ret = -ENOMEM;
1915  		goto out_release;
1916  	}
1917  
1918  	INIT_WORK(&queue->release_work, nvmet_tcp_release_queue_work);
1919  	INIT_WORK(&queue->io_work, nvmet_tcp_io_work);
1920  	kref_init(&queue->kref);
1921  	queue->sock = newsock;
1922  	queue->port = port;
1923  	queue->nr_cmds = 0;
1924  	spin_lock_init(&queue->state_lock);
1925  	if (queue->port->nport->disc_addr.tsas.tcp.sectype ==
1926  	    NVMF_TCP_SECTYPE_TLS13)
1927  		queue->state = NVMET_TCP_Q_TLS_HANDSHAKE;
1928  	else
1929  		queue->state = NVMET_TCP_Q_CONNECTING;
1930  	INIT_LIST_HEAD(&queue->free_list);
1931  	init_llist_head(&queue->resp_list);
1932  	INIT_LIST_HEAD(&queue->resp_send_list);
1933  
1934  	sock_file = sock_alloc_file(queue->sock, O_CLOEXEC, NULL);
1935  	if (IS_ERR(sock_file)) {
1936  		ret = PTR_ERR(sock_file);
1937  		goto out_free_queue;
1938  	}
1939  
1940  	queue->idx = ida_alloc(&nvmet_tcp_queue_ida, GFP_KERNEL);
1941  	if (queue->idx < 0) {
1942  		ret = queue->idx;
1943  		goto out_sock;
1944  	}
1945  
1946  	ret = nvmet_tcp_alloc_cmd(queue, &queue->connect);
1947  	if (ret)
1948  		goto out_ida_remove;
1949  
1950  	ret = nvmet_sq_init(&queue->nvme_sq);
1951  	if (ret)
1952  		goto out_free_connect;
1953  
1954  	nvmet_prepare_receive_pdu(queue);
1955  
1956  	mutex_lock(&nvmet_tcp_queue_mutex);
1957  	list_add_tail(&queue->queue_list, &nvmet_tcp_queue_list);
1958  	mutex_unlock(&nvmet_tcp_queue_mutex);
1959  
1960  	INIT_DELAYED_WORK(&queue->tls_handshake_tmo_work,
1961  			  nvmet_tcp_tls_handshake_timeout);
1962  #ifdef CONFIG_NVME_TARGET_TCP_TLS
1963  	if (queue->state == NVMET_TCP_Q_TLS_HANDSHAKE) {
1964  		struct sock *sk = queue->sock->sk;
1965  
1966  		/* Restore the default callbacks before starting upcall */
1967  		read_lock_bh(&sk->sk_callback_lock);
1968  		sk->sk_user_data = NULL;
1969  		sk->sk_data_ready = port->data_ready;
1970  		read_unlock_bh(&sk->sk_callback_lock);
1971  		if (!nvmet_tcp_try_peek_pdu(queue)) {
1972  			if (!nvmet_tcp_tls_handshake(queue))
1973  				return;
1974  			/* TLS handshake failed, terminate the connection */
1975  			goto out_destroy_sq;
1976  		}
1977  		/* Not a TLS connection, continue with normal processing */
1978  		queue->state = NVMET_TCP_Q_CONNECTING;
1979  	}
1980  #endif
1981  
1982  	ret = nvmet_tcp_set_queue_sock(queue);
1983  	if (ret)
1984  		goto out_destroy_sq;
1985  
1986  	return;
1987  out_destroy_sq:
1988  	mutex_lock(&nvmet_tcp_queue_mutex);
1989  	list_del_init(&queue->queue_list);
1990  	mutex_unlock(&nvmet_tcp_queue_mutex);
1991  	nvmet_sq_destroy(&queue->nvme_sq);
1992  out_free_connect:
1993  	nvmet_tcp_free_cmd(&queue->connect);
1994  out_ida_remove:
1995  	ida_free(&nvmet_tcp_queue_ida, queue->idx);
1996  out_sock:
1997  	fput(queue->sock->file);
1998  out_free_queue:
1999  	kfree(queue);
2000  out_release:
2001  	pr_err("failed to allocate queue, error %d\n", ret);
2002  	if (!sock_file)
2003  		sock_release(newsock);
2004  }
2005  
nvmet_tcp_accept_work(struct work_struct * w)2006  static void nvmet_tcp_accept_work(struct work_struct *w)
2007  {
2008  	struct nvmet_tcp_port *port =
2009  		container_of(w, struct nvmet_tcp_port, accept_work);
2010  	struct socket *newsock;
2011  	int ret;
2012  
2013  	while (true) {
2014  		ret = kernel_accept(port->sock, &newsock, O_NONBLOCK);
2015  		if (ret < 0) {
2016  			if (ret != -EAGAIN)
2017  				pr_warn("failed to accept err=%d\n", ret);
2018  			return;
2019  		}
2020  		nvmet_tcp_alloc_queue(port, newsock);
2021  	}
2022  }
2023  
nvmet_tcp_listen_data_ready(struct sock * sk)2024  static void nvmet_tcp_listen_data_ready(struct sock *sk)
2025  {
2026  	struct nvmet_tcp_port *port;
2027  
2028  	trace_sk_data_ready(sk);
2029  
2030  	read_lock_bh(&sk->sk_callback_lock);
2031  	port = sk->sk_user_data;
2032  	if (!port)
2033  		goto out;
2034  
2035  	if (sk->sk_state == TCP_LISTEN)
2036  		queue_work(nvmet_wq, &port->accept_work);
2037  out:
2038  	read_unlock_bh(&sk->sk_callback_lock);
2039  }
2040  
nvmet_tcp_add_port(struct nvmet_port * nport)2041  static int nvmet_tcp_add_port(struct nvmet_port *nport)
2042  {
2043  	struct nvmet_tcp_port *port;
2044  	__kernel_sa_family_t af;
2045  	int ret;
2046  
2047  	port = kzalloc(sizeof(*port), GFP_KERNEL);
2048  	if (!port)
2049  		return -ENOMEM;
2050  
2051  	switch (nport->disc_addr.adrfam) {
2052  	case NVMF_ADDR_FAMILY_IP4:
2053  		af = AF_INET;
2054  		break;
2055  	case NVMF_ADDR_FAMILY_IP6:
2056  		af = AF_INET6;
2057  		break;
2058  	default:
2059  		pr_err("address family %d not supported\n",
2060  				nport->disc_addr.adrfam);
2061  		ret = -EINVAL;
2062  		goto err_port;
2063  	}
2064  
2065  	ret = inet_pton_with_scope(&init_net, af, nport->disc_addr.traddr,
2066  			nport->disc_addr.trsvcid, &port->addr);
2067  	if (ret) {
2068  		pr_err("malformed ip/port passed: %s:%s\n",
2069  			nport->disc_addr.traddr, nport->disc_addr.trsvcid);
2070  		goto err_port;
2071  	}
2072  
2073  	port->nport = nport;
2074  	INIT_WORK(&port->accept_work, nvmet_tcp_accept_work);
2075  	if (port->nport->inline_data_size < 0)
2076  		port->nport->inline_data_size = NVMET_TCP_DEF_INLINE_DATA_SIZE;
2077  
2078  	ret = sock_create(port->addr.ss_family, SOCK_STREAM,
2079  				IPPROTO_TCP, &port->sock);
2080  	if (ret) {
2081  		pr_err("failed to create a socket\n");
2082  		goto err_port;
2083  	}
2084  
2085  	port->sock->sk->sk_user_data = port;
2086  	port->data_ready = port->sock->sk->sk_data_ready;
2087  	port->sock->sk->sk_data_ready = nvmet_tcp_listen_data_ready;
2088  	sock_set_reuseaddr(port->sock->sk);
2089  	tcp_sock_set_nodelay(port->sock->sk);
2090  	if (so_priority > 0)
2091  		sock_set_priority(port->sock->sk, so_priority);
2092  
2093  	ret = kernel_bind(port->sock, (struct sockaddr *)&port->addr,
2094  			sizeof(port->addr));
2095  	if (ret) {
2096  		pr_err("failed to bind port socket %d\n", ret);
2097  		goto err_sock;
2098  	}
2099  
2100  	ret = kernel_listen(port->sock, NVMET_TCP_BACKLOG);
2101  	if (ret) {
2102  		pr_err("failed to listen %d on port sock\n", ret);
2103  		goto err_sock;
2104  	}
2105  
2106  	nport->priv = port;
2107  	pr_info("enabling port %d (%pISpc)\n",
2108  		le16_to_cpu(nport->disc_addr.portid), &port->addr);
2109  
2110  	return 0;
2111  
2112  err_sock:
2113  	sock_release(port->sock);
2114  err_port:
2115  	kfree(port);
2116  	return ret;
2117  }
2118  
nvmet_tcp_destroy_port_queues(struct nvmet_tcp_port * port)2119  static void nvmet_tcp_destroy_port_queues(struct nvmet_tcp_port *port)
2120  {
2121  	struct nvmet_tcp_queue *queue;
2122  
2123  	mutex_lock(&nvmet_tcp_queue_mutex);
2124  	list_for_each_entry(queue, &nvmet_tcp_queue_list, queue_list)
2125  		if (queue->port == port)
2126  			kernel_sock_shutdown(queue->sock, SHUT_RDWR);
2127  	mutex_unlock(&nvmet_tcp_queue_mutex);
2128  }
2129  
nvmet_tcp_remove_port(struct nvmet_port * nport)2130  static void nvmet_tcp_remove_port(struct nvmet_port *nport)
2131  {
2132  	struct nvmet_tcp_port *port = nport->priv;
2133  
2134  	write_lock_bh(&port->sock->sk->sk_callback_lock);
2135  	port->sock->sk->sk_data_ready = port->data_ready;
2136  	port->sock->sk->sk_user_data = NULL;
2137  	write_unlock_bh(&port->sock->sk->sk_callback_lock);
2138  	cancel_work_sync(&port->accept_work);
2139  	/*
2140  	 * Destroy the remaining queues, which are not belong to any
2141  	 * controller yet.
2142  	 */
2143  	nvmet_tcp_destroy_port_queues(port);
2144  
2145  	sock_release(port->sock);
2146  	kfree(port);
2147  }
2148  
nvmet_tcp_delete_ctrl(struct nvmet_ctrl * ctrl)2149  static void nvmet_tcp_delete_ctrl(struct nvmet_ctrl *ctrl)
2150  {
2151  	struct nvmet_tcp_queue *queue;
2152  
2153  	mutex_lock(&nvmet_tcp_queue_mutex);
2154  	list_for_each_entry(queue, &nvmet_tcp_queue_list, queue_list)
2155  		if (queue->nvme_sq.ctrl == ctrl)
2156  			kernel_sock_shutdown(queue->sock, SHUT_RDWR);
2157  	mutex_unlock(&nvmet_tcp_queue_mutex);
2158  }
2159  
nvmet_tcp_install_queue(struct nvmet_sq * sq)2160  static u16 nvmet_tcp_install_queue(struct nvmet_sq *sq)
2161  {
2162  	struct nvmet_tcp_queue *queue =
2163  		container_of(sq, struct nvmet_tcp_queue, nvme_sq);
2164  
2165  	if (sq->qid == 0) {
2166  		struct nvmet_tcp_queue *q;
2167  		int pending = 0;
2168  
2169  		/* Check for pending controller teardown */
2170  		mutex_lock(&nvmet_tcp_queue_mutex);
2171  		list_for_each_entry(q, &nvmet_tcp_queue_list, queue_list) {
2172  			if (q->nvme_sq.ctrl == sq->ctrl &&
2173  			    q->state == NVMET_TCP_Q_DISCONNECTING)
2174  				pending++;
2175  		}
2176  		mutex_unlock(&nvmet_tcp_queue_mutex);
2177  		if (pending > NVMET_TCP_BACKLOG)
2178  			return NVME_SC_CONNECT_CTRL_BUSY;
2179  	}
2180  
2181  	queue->nr_cmds = sq->size * 2;
2182  	if (nvmet_tcp_alloc_cmds(queue)) {
2183  		queue->nr_cmds = 0;
2184  		return NVME_SC_INTERNAL;
2185  	}
2186  	return 0;
2187  }
2188  
nvmet_tcp_disc_port_addr(struct nvmet_req * req,struct nvmet_port * nport,char * traddr)2189  static void nvmet_tcp_disc_port_addr(struct nvmet_req *req,
2190  		struct nvmet_port *nport, char *traddr)
2191  {
2192  	struct nvmet_tcp_port *port = nport->priv;
2193  
2194  	if (inet_addr_is_any((struct sockaddr *)&port->addr)) {
2195  		struct nvmet_tcp_cmd *cmd =
2196  			container_of(req, struct nvmet_tcp_cmd, req);
2197  		struct nvmet_tcp_queue *queue = cmd->queue;
2198  
2199  		sprintf(traddr, "%pISc", (struct sockaddr *)&queue->sockaddr);
2200  	} else {
2201  		memcpy(traddr, nport->disc_addr.traddr, NVMF_TRADDR_SIZE);
2202  	}
2203  }
2204  
nvmet_tcp_host_port_addr(struct nvmet_ctrl * ctrl,char * traddr,size_t traddr_len)2205  static ssize_t nvmet_tcp_host_port_addr(struct nvmet_ctrl *ctrl,
2206  			char *traddr, size_t traddr_len)
2207  {
2208  	struct nvmet_sq *sq = ctrl->sqs[0];
2209  	struct nvmet_tcp_queue *queue =
2210  		container_of(sq, struct nvmet_tcp_queue, nvme_sq);
2211  
2212  	if (queue->sockaddr_peer.ss_family == AF_UNSPEC)
2213  		return -EINVAL;
2214  	return snprintf(traddr, traddr_len, "%pISc",
2215  			(struct sockaddr *)&queue->sockaddr_peer);
2216  }
2217  
2218  static const struct nvmet_fabrics_ops nvmet_tcp_ops = {
2219  	.owner			= THIS_MODULE,
2220  	.type			= NVMF_TRTYPE_TCP,
2221  	.msdbd			= 1,
2222  	.add_port		= nvmet_tcp_add_port,
2223  	.remove_port		= nvmet_tcp_remove_port,
2224  	.queue_response		= nvmet_tcp_queue_response,
2225  	.delete_ctrl		= nvmet_tcp_delete_ctrl,
2226  	.install_queue		= nvmet_tcp_install_queue,
2227  	.disc_traddr		= nvmet_tcp_disc_port_addr,
2228  	.host_traddr		= nvmet_tcp_host_port_addr,
2229  };
2230  
nvmet_tcp_init(void)2231  static int __init nvmet_tcp_init(void)
2232  {
2233  	int ret;
2234  
2235  	nvmet_tcp_wq = alloc_workqueue("nvmet_tcp_wq",
2236  				WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
2237  	if (!nvmet_tcp_wq)
2238  		return -ENOMEM;
2239  
2240  	ret = nvmet_register_transport(&nvmet_tcp_ops);
2241  	if (ret)
2242  		goto err;
2243  
2244  	return 0;
2245  err:
2246  	destroy_workqueue(nvmet_tcp_wq);
2247  	return ret;
2248  }
2249  
nvmet_tcp_exit(void)2250  static void __exit nvmet_tcp_exit(void)
2251  {
2252  	struct nvmet_tcp_queue *queue;
2253  
2254  	nvmet_unregister_transport(&nvmet_tcp_ops);
2255  
2256  	flush_workqueue(nvmet_wq);
2257  	mutex_lock(&nvmet_tcp_queue_mutex);
2258  	list_for_each_entry(queue, &nvmet_tcp_queue_list, queue_list)
2259  		kernel_sock_shutdown(queue->sock, SHUT_RDWR);
2260  	mutex_unlock(&nvmet_tcp_queue_mutex);
2261  	flush_workqueue(nvmet_wq);
2262  
2263  	destroy_workqueue(nvmet_tcp_wq);
2264  	ida_destroy(&nvmet_tcp_queue_ida);
2265  }
2266  
2267  module_init(nvmet_tcp_init);
2268  module_exit(nvmet_tcp_exit);
2269  
2270  MODULE_DESCRIPTION("NVMe target TCP transport driver");
2271  MODULE_LICENSE("GPL v2");
2272  MODULE_ALIAS("nvmet-transport-3"); /* 3 == NVMF_TRTYPE_TCP */
2273