1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * NVMe over Fabrics TCP target. 4 * Copyright (c) 2018 Lightbits Labs. All rights reserved. 5 */ 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 #include <linux/module.h> 8 #include <linux/init.h> 9 #include <linux/slab.h> 10 #include <linux/crc32c.h> 11 #include <linux/err.h> 12 #include <linux/nvme-tcp.h> 13 #include <linux/nvme-keyring.h> 14 #include <net/sock.h> 15 #include <net/tcp.h> 16 #include <net/tls.h> 17 #include <net/tls_prot.h> 18 #include <net/handshake.h> 19 #include <linux/inet.h> 20 #include <linux/llist.h> 21 #include <trace/events/sock.h> 22 23 #include "nvmet.h" 24 25 #define NVMET_TCP_DEF_INLINE_DATA_SIZE (4 * PAGE_SIZE) 26 #define NVMET_TCP_MAXH2CDATA 0x400000 /* 16M arbitrary limit */ 27 #define NVMET_TCP_BACKLOG 128 28 29 static int param_store_val(const char *str, int *val, int min, int max) 30 { 31 int ret, new_val; 32 33 ret = kstrtoint(str, 10, &new_val); 34 if (ret) 35 return -EINVAL; 36 37 if (new_val < min || new_val > max) 38 return -EINVAL; 39 40 *val = new_val; 41 return 0; 42 } 43 44 static int set_params(const char *str, const struct kernel_param *kp) 45 { 46 return param_store_val(str, kp->arg, 0, INT_MAX); 47 } 48 49 static const struct kernel_param_ops set_param_ops = { 50 .set = set_params, 51 .get = param_get_int, 52 }; 53 54 /* Define the socket priority to use for connections were it is desirable 55 * that the NIC consider performing optimized packet processing or filtering. 56 * A non-zero value being sufficient to indicate general consideration of any 57 * possible optimization. Making it a module param allows for alternative 58 * values that may be unique for some NIC implementations. 59 */ 60 static int so_priority; 61 device_param_cb(so_priority, &set_param_ops, &so_priority, 0644); 62 MODULE_PARM_DESC(so_priority, "nvmet tcp socket optimize priority: Default 0"); 63 64 /* Define a time period (in usecs) that io_work() shall sample an activated 65 * queue before determining it to be idle. This optional module behavior 66 * can enable NIC solutions that support socket optimized packet processing 67 * using advanced interrupt moderation techniques. 68 */ 69 static int idle_poll_period_usecs; 70 device_param_cb(idle_poll_period_usecs, &set_param_ops, 71 &idle_poll_period_usecs, 0644); 72 MODULE_PARM_DESC(idle_poll_period_usecs, 73 "nvmet tcp io_work poll till idle time period in usecs: Default 0"); 74 75 #ifdef CONFIG_NVME_TARGET_TCP_TLS 76 /* 77 * TLS handshake timeout 78 */ 79 static int tls_handshake_timeout = 10; 80 module_param(tls_handshake_timeout, int, 0644); 81 MODULE_PARM_DESC(tls_handshake_timeout, 82 "nvme TLS handshake timeout in seconds (default 10)"); 83 #endif 84 85 #define NVMET_TCP_RECV_BUDGET 8 86 #define NVMET_TCP_SEND_BUDGET 8 87 #define NVMET_TCP_IO_WORK_BUDGET 64 88 89 enum nvmet_tcp_send_state { 90 NVMET_TCP_SEND_DATA_PDU, 91 NVMET_TCP_SEND_DATA, 92 NVMET_TCP_SEND_R2T, 93 NVMET_TCP_SEND_DDGST, 94 NVMET_TCP_SEND_RESPONSE 95 }; 96 97 enum nvmet_tcp_recv_state { 98 NVMET_TCP_RECV_PDU, 99 NVMET_TCP_RECV_DATA, 100 NVMET_TCP_RECV_DDGST, 101 NVMET_TCP_RECV_ERR, 102 }; 103 104 enum { 105 NVMET_TCP_F_INIT_FAILED = (1 << 0), 106 }; 107 108 struct nvmet_tcp_cmd { 109 struct nvmet_tcp_queue *queue; 110 struct nvmet_req req; 111 112 struct nvme_tcp_cmd_pdu *cmd_pdu; 113 struct nvme_tcp_rsp_pdu *rsp_pdu; 114 struct nvme_tcp_data_pdu *data_pdu; 115 struct nvme_tcp_r2t_pdu *r2t_pdu; 116 117 u32 rbytes_done; 118 u32 wbytes_done; 119 120 u32 pdu_len; 121 u32 pdu_recv; 122 int sg_idx; 123 char recv_cbuf[CMSG_LEN(sizeof(char))]; 124 struct msghdr recv_msg; 125 struct bio_vec *iov; 126 u32 flags; 127 128 struct list_head entry; 129 struct llist_node lentry; 130 131 /* send state */ 132 u32 offset; 133 struct scatterlist *cur_sg; 134 enum nvmet_tcp_send_state state; 135 136 __le32 exp_ddgst; 137 __le32 recv_ddgst; 138 }; 139 140 enum nvmet_tcp_queue_state { 141 NVMET_TCP_Q_CONNECTING, 142 NVMET_TCP_Q_TLS_HANDSHAKE, 143 NVMET_TCP_Q_LIVE, 144 NVMET_TCP_Q_DISCONNECTING, 145 NVMET_TCP_Q_FAILED, 146 }; 147 148 struct nvmet_tcp_queue { 149 struct socket *sock; 150 struct nvmet_tcp_port *port; 151 struct work_struct io_work; 152 struct nvmet_cq nvme_cq; 153 struct nvmet_sq nvme_sq; 154 struct kref kref; 155 156 /* send state */ 157 struct nvmet_tcp_cmd *cmds; 158 unsigned int nr_cmds; 159 struct list_head free_list; 160 struct llist_head resp_list; 161 struct list_head resp_send_list; 162 int send_list_len; 163 struct nvmet_tcp_cmd *snd_cmd; 164 165 /* recv state */ 166 int offset; 167 int left; 168 enum nvmet_tcp_recv_state rcv_state; 169 struct nvmet_tcp_cmd *cmd; 170 union nvme_tcp_pdu pdu; 171 172 /* digest state */ 173 bool hdr_digest; 174 bool data_digest; 175 176 /* TLS state */ 177 key_serial_t tls_pskid; 178 struct delayed_work tls_handshake_tmo_work; 179 180 unsigned long poll_end; 181 182 spinlock_t state_lock; 183 enum nvmet_tcp_queue_state state; 184 185 struct sockaddr_storage sockaddr; 186 struct sockaddr_storage sockaddr_peer; 187 struct work_struct release_work; 188 189 int idx; 190 struct list_head queue_list; 191 192 struct nvmet_tcp_cmd connect; 193 194 struct page_frag_cache pf_cache; 195 196 void (*data_ready)(struct sock *); 197 void (*state_change)(struct sock *); 198 void (*write_space)(struct sock *); 199 }; 200 201 struct nvmet_tcp_port { 202 struct socket *sock; 203 struct work_struct accept_work; 204 struct nvmet_port *nport; 205 struct sockaddr_storage addr; 206 void (*data_ready)(struct sock *); 207 }; 208 209 static DEFINE_IDA(nvmet_tcp_queue_ida); 210 static LIST_HEAD(nvmet_tcp_queue_list); 211 static DEFINE_MUTEX(nvmet_tcp_queue_mutex); 212 213 static struct workqueue_struct *nvmet_tcp_wq; 214 static const struct nvmet_fabrics_ops nvmet_tcp_ops; 215 static void nvmet_tcp_free_cmd(struct nvmet_tcp_cmd *c); 216 static void nvmet_tcp_free_cmd_buffers(struct nvmet_tcp_cmd *cmd); 217 218 static inline u16 nvmet_tcp_cmd_tag(struct nvmet_tcp_queue *queue, 219 struct nvmet_tcp_cmd *cmd) 220 { 221 if (unlikely(!queue->nr_cmds)) { 222 /* We didn't allocate cmds yet, send 0xffff */ 223 return USHRT_MAX; 224 } 225 226 return cmd - queue->cmds; 227 } 228 229 static inline bool nvmet_tcp_has_data_in(struct nvmet_tcp_cmd *cmd) 230 { 231 return nvme_is_write(cmd->req.cmd) && 232 cmd->rbytes_done < cmd->req.transfer_len; 233 } 234 235 static inline bool nvmet_tcp_need_data_in(struct nvmet_tcp_cmd *cmd) 236 { 237 return nvmet_tcp_has_data_in(cmd) && !cmd->req.cqe->status; 238 } 239 240 static inline bool nvmet_tcp_need_data_out(struct nvmet_tcp_cmd *cmd) 241 { 242 return !nvme_is_write(cmd->req.cmd) && 243 cmd->req.transfer_len > 0 && 244 !cmd->req.cqe->status; 245 } 246 247 static inline bool nvmet_tcp_has_inline_data(struct nvmet_tcp_cmd *cmd) 248 { 249 return nvme_is_write(cmd->req.cmd) && cmd->pdu_len && 250 !cmd->rbytes_done; 251 } 252 253 static inline struct nvmet_tcp_cmd * 254 nvmet_tcp_get_cmd(struct nvmet_tcp_queue *queue) 255 { 256 struct nvmet_tcp_cmd *cmd; 257 258 cmd = list_first_entry_or_null(&queue->free_list, 259 struct nvmet_tcp_cmd, entry); 260 if (!cmd) 261 return NULL; 262 list_del_init(&cmd->entry); 263 264 cmd->rbytes_done = cmd->wbytes_done = 0; 265 cmd->pdu_len = 0; 266 cmd->pdu_recv = 0; 267 cmd->iov = NULL; 268 cmd->flags = 0; 269 return cmd; 270 } 271 272 static inline void nvmet_tcp_put_cmd(struct nvmet_tcp_cmd *cmd) 273 { 274 if (unlikely(cmd == &cmd->queue->connect)) 275 return; 276 277 list_add_tail(&cmd->entry, &cmd->queue->free_list); 278 } 279 280 static inline int queue_cpu(struct nvmet_tcp_queue *queue) 281 { 282 return queue->sock->sk->sk_incoming_cpu; 283 } 284 285 static inline u8 nvmet_tcp_hdgst_len(struct nvmet_tcp_queue *queue) 286 { 287 return queue->hdr_digest ? NVME_TCP_DIGEST_LENGTH : 0; 288 } 289 290 static inline u8 nvmet_tcp_ddgst_len(struct nvmet_tcp_queue *queue) 291 { 292 return queue->data_digest ? NVME_TCP_DIGEST_LENGTH : 0; 293 } 294 295 static inline void nvmet_tcp_hdgst(void *pdu, size_t len) 296 { 297 put_unaligned_le32(~crc32c(~0, pdu, len), pdu + len); 298 } 299 300 static int nvmet_tcp_verify_hdgst(struct nvmet_tcp_queue *queue, 301 void *pdu, size_t len) 302 { 303 struct nvme_tcp_hdr *hdr = pdu; 304 __le32 recv_digest; 305 __le32 exp_digest; 306 307 if (unlikely(!(hdr->flags & NVME_TCP_F_HDGST))) { 308 pr_err("queue %d: header digest enabled but no header digest\n", 309 queue->idx); 310 return -EPROTO; 311 } 312 313 recv_digest = *(__le32 *)(pdu + hdr->hlen); 314 nvmet_tcp_hdgst(pdu, len); 315 exp_digest = *(__le32 *)(pdu + hdr->hlen); 316 if (recv_digest != exp_digest) { 317 pr_err("queue %d: header digest error: recv %#x expected %#x\n", 318 queue->idx, le32_to_cpu(recv_digest), 319 le32_to_cpu(exp_digest)); 320 return -EPROTO; 321 } 322 323 return 0; 324 } 325 326 static int nvmet_tcp_check_ddgst(struct nvmet_tcp_queue *queue, void *pdu) 327 { 328 struct nvme_tcp_hdr *hdr = pdu; 329 u8 digest_len = nvmet_tcp_hdgst_len(queue); 330 u32 len; 331 332 len = le32_to_cpu(hdr->plen) - hdr->hlen - 333 (hdr->flags & NVME_TCP_F_HDGST ? digest_len : 0); 334 335 if (unlikely(len && !(hdr->flags & NVME_TCP_F_DDGST))) { 336 pr_err("queue %d: data digest flag is cleared\n", queue->idx); 337 return -EPROTO; 338 } 339 340 return 0; 341 } 342 343 /* If cmd buffers are NULL, no operation is performed */ 344 static void nvmet_tcp_free_cmd_buffers(struct nvmet_tcp_cmd *cmd) 345 { 346 kfree(cmd->iov); 347 sgl_free(cmd->req.sg); 348 cmd->iov = NULL; 349 cmd->req.sg = NULL; 350 } 351 352 static int nvmet_tcp_build_pdu_iovec(struct nvmet_tcp_cmd *cmd) 353 { 354 struct bio_vec *iov = cmd->iov; 355 struct scatterlist *sg; 356 u32 length, offset, sg_offset; 357 unsigned int sg_remaining; 358 int nr_pages; 359 360 length = cmd->pdu_len; 361 nr_pages = DIV_ROUND_UP(length, PAGE_SIZE); 362 offset = cmd->rbytes_done; 363 cmd->sg_idx = offset / PAGE_SIZE; 364 sg_offset = offset % PAGE_SIZE; 365 if (!cmd->req.sg_cnt || cmd->sg_idx >= cmd->req.sg_cnt) 366 return -EPROTO; 367 368 sg = &cmd->req.sg[cmd->sg_idx]; 369 sg_remaining = cmd->req.sg_cnt - cmd->sg_idx; 370 371 while (length) { 372 if (!sg_remaining) 373 return -EPROTO; 374 375 if (!sg->length || sg->length <= sg_offset) 376 return -EPROTO; 377 378 u32 iov_len = min_t(u32, length, sg->length - sg_offset); 379 380 bvec_set_page(iov, sg_page(sg), iov_len, 381 sg->offset + sg_offset); 382 383 length -= iov_len; 384 sg = sg_next(sg); 385 sg_remaining--; 386 iov++; 387 sg_offset = 0; 388 } 389 390 iov_iter_bvec(&cmd->recv_msg.msg_iter, ITER_DEST, cmd->iov, 391 nr_pages, cmd->pdu_len); 392 return 0; 393 } 394 395 static void nvmet_tcp_socket_error(struct nvmet_tcp_queue *queue, int status) 396 { 397 /* 398 * Keep rcv_state at RECV_ERR even for the internal -ESHUTDOWN path. 399 * nvmet_tcp_handle_icreq() can return -ESHUTDOWN after the ICReq has 400 * already been consumed and queue teardown has started. 401 * 402 * If nvmet_tcp_data_ready() or nvmet_tcp_write_space() queues 403 * nvmet_tcp_io_work() again before nvmet_tcp_release_queue_work() 404 * cancels it, the queue must not keep that old receive state. 405 * Otherwise the next nvmet_tcp_io_work() run can reach 406 * nvmet_tcp_done_recv_pdu() and try to handle the same ICReq again. 407 * 408 * That is why queue->rcv_state needs to be updated before we return. 409 */ 410 queue->rcv_state = NVMET_TCP_RECV_ERR; 411 if (status == -EPIPE || status == -ECONNRESET || !queue->nvme_sq.ctrl) 412 kernel_sock_shutdown(queue->sock, SHUT_RDWR); 413 else 414 nvmet_ctrl_fatal_error(queue->nvme_sq.ctrl); 415 } 416 417 static int nvmet_tcp_map_data(struct nvmet_tcp_cmd *cmd) 418 { 419 struct nvme_sgl_desc *sgl = &cmd->req.cmd->common.dptr.sgl; 420 u32 len = le32_to_cpu(sgl->length); 421 422 if (!len) 423 return 0; 424 425 /* 426 * inline_data_size only bounds the in-capsule (type 0x01) SGL 427 * descriptor below. A non-inline transport SGL data-block 428 * descriptor skips that check entirely and would otherwise reach 429 * sgl_alloc() with an attacker-controlled len of up to 4 GiB, 430 * pinning that much kernel memory for a command that may never 431 * complete. Bound every descriptor type here, before allocating 432 * anything, using the same ceiling this file already applies to 433 * per-PDU H2C data. 434 */ 435 if (len > NVMET_TCP_MAXH2CDATA) 436 return NVME_SC_SGL_INVALID_DATA | NVME_STATUS_DNR; 437 438 if (sgl->type == ((NVME_SGL_FMT_DATA_DESC << 4) | 439 NVME_SGL_FMT_OFFSET)) { 440 if (!nvme_is_write(cmd->req.cmd)) 441 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 442 443 if (len > cmd->req.port->inline_data_size) 444 return NVME_SC_SGL_INVALID_OFFSET | NVME_STATUS_DNR; 445 cmd->pdu_len = len; 446 } 447 cmd->req.transfer_len += len; 448 449 cmd->req.sg = sgl_alloc(len, GFP_KERNEL | __GFP_NOWARN, 450 &cmd->req.sg_cnt); 451 if (!cmd->req.sg) 452 return NVME_SC_INTERNAL; 453 cmd->cur_sg = cmd->req.sg; 454 455 if (nvmet_tcp_has_data_in(cmd)) { 456 cmd->iov = kmalloc_objs(*cmd->iov, cmd->req.sg_cnt, 457 GFP_KERNEL | __GFP_NOWARN); 458 if (!cmd->iov) 459 goto err; 460 } 461 462 return 0; 463 err: 464 nvmet_tcp_free_cmd_buffers(cmd); 465 return NVME_SC_INTERNAL; 466 } 467 468 static void nvmet_tcp_calc_ddgst(struct nvmet_tcp_cmd *cmd) 469 { 470 size_t total_len = cmd->req.transfer_len; 471 struct scatterlist *sg = cmd->req.sg; 472 u32 crc = ~0; 473 474 while (total_len) { 475 size_t len = min_t(size_t, total_len, sg->length); 476 477 /* 478 * Note that the scatterlist does not contain any highmem pages, 479 * as it was allocated by sgl_alloc() with GFP_KERNEL. 480 */ 481 crc = crc32c(crc, sg_virt(sg), len); 482 total_len -= len; 483 sg = sg_next(sg); 484 } 485 cmd->exp_ddgst = cpu_to_le32(~crc); 486 } 487 488 static void nvmet_setup_c2h_data_pdu(struct nvmet_tcp_cmd *cmd) 489 { 490 struct nvme_tcp_data_pdu *pdu = cmd->data_pdu; 491 struct nvmet_tcp_queue *queue = cmd->queue; 492 u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue); 493 u8 ddgst = nvmet_tcp_ddgst_len(cmd->queue); 494 495 cmd->offset = 0; 496 cmd->state = NVMET_TCP_SEND_DATA_PDU; 497 498 pdu->hdr.type = nvme_tcp_c2h_data; 499 pdu->hdr.flags = NVME_TCP_F_DATA_LAST | (queue->nvme_sq.sqhd_disabled ? 500 NVME_TCP_F_DATA_SUCCESS : 0); 501 pdu->hdr.hlen = sizeof(*pdu); 502 pdu->hdr.pdo = pdu->hdr.hlen + hdgst; 503 pdu->hdr.plen = 504 cpu_to_le32(pdu->hdr.hlen + hdgst + 505 cmd->req.transfer_len + ddgst); 506 pdu->command_id = cmd->req.cqe->command_id; 507 pdu->data_length = cpu_to_le32(cmd->req.transfer_len); 508 pdu->data_offset = cpu_to_le32(cmd->wbytes_done); 509 510 if (queue->data_digest) { 511 pdu->hdr.flags |= NVME_TCP_F_DDGST; 512 nvmet_tcp_calc_ddgst(cmd); 513 } 514 515 if (cmd->queue->hdr_digest) { 516 pdu->hdr.flags |= NVME_TCP_F_HDGST; 517 nvmet_tcp_hdgst(pdu, sizeof(*pdu)); 518 } 519 } 520 521 static void nvmet_setup_r2t_pdu(struct nvmet_tcp_cmd *cmd) 522 { 523 struct nvme_tcp_r2t_pdu *pdu = cmd->r2t_pdu; 524 u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue); 525 526 cmd->offset = 0; 527 cmd->state = NVMET_TCP_SEND_R2T; 528 529 pdu->hdr.type = nvme_tcp_r2t; 530 pdu->hdr.flags = 0; 531 pdu->hdr.hlen = sizeof(*pdu); 532 pdu->hdr.pdo = 0; 533 pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst); 534 535 pdu->command_id = cmd->req.cmd->common.command_id; 536 pdu->ttag = nvmet_tcp_cmd_tag(cmd->queue, cmd); 537 pdu->r2t_length = cpu_to_le32(cmd->req.transfer_len - cmd->rbytes_done); 538 pdu->r2t_offset = cpu_to_le32(cmd->rbytes_done); 539 if (cmd->queue->hdr_digest) { 540 pdu->hdr.flags |= NVME_TCP_F_HDGST; 541 nvmet_tcp_hdgst(pdu, sizeof(*pdu)); 542 } 543 } 544 545 static void nvmet_setup_response_pdu(struct nvmet_tcp_cmd *cmd) 546 { 547 struct nvme_tcp_rsp_pdu *pdu = cmd->rsp_pdu; 548 u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue); 549 550 cmd->offset = 0; 551 cmd->state = NVMET_TCP_SEND_RESPONSE; 552 553 pdu->hdr.type = nvme_tcp_rsp; 554 pdu->hdr.flags = 0; 555 pdu->hdr.hlen = sizeof(*pdu); 556 pdu->hdr.pdo = 0; 557 pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst); 558 if (cmd->queue->hdr_digest) { 559 pdu->hdr.flags |= NVME_TCP_F_HDGST; 560 nvmet_tcp_hdgst(pdu, sizeof(*pdu)); 561 } 562 } 563 564 static void nvmet_tcp_process_resp_list(struct nvmet_tcp_queue *queue) 565 { 566 struct llist_node *node; 567 struct nvmet_tcp_cmd *cmd; 568 569 for (node = llist_del_all(&queue->resp_list); node; node = node->next) { 570 cmd = llist_entry(node, struct nvmet_tcp_cmd, lentry); 571 list_add(&cmd->entry, &queue->resp_send_list); 572 queue->send_list_len++; 573 } 574 } 575 576 static struct nvmet_tcp_cmd *nvmet_tcp_fetch_cmd(struct nvmet_tcp_queue *queue) 577 { 578 queue->snd_cmd = list_first_entry_or_null(&queue->resp_send_list, 579 struct nvmet_tcp_cmd, entry); 580 if (!queue->snd_cmd) { 581 nvmet_tcp_process_resp_list(queue); 582 queue->snd_cmd = 583 list_first_entry_or_null(&queue->resp_send_list, 584 struct nvmet_tcp_cmd, entry); 585 if (unlikely(!queue->snd_cmd)) 586 return NULL; 587 } 588 589 list_del_init(&queue->snd_cmd->entry); 590 queue->send_list_len--; 591 592 if (nvmet_tcp_need_data_out(queue->snd_cmd)) 593 nvmet_setup_c2h_data_pdu(queue->snd_cmd); 594 else if (nvmet_tcp_need_data_in(queue->snd_cmd)) 595 nvmet_setup_r2t_pdu(queue->snd_cmd); 596 else 597 nvmet_setup_response_pdu(queue->snd_cmd); 598 599 return queue->snd_cmd; 600 } 601 602 static void nvmet_tcp_queue_response(struct nvmet_req *req) 603 { 604 struct nvmet_tcp_cmd *cmd = 605 container_of(req, struct nvmet_tcp_cmd, req); 606 struct nvmet_tcp_queue *queue = cmd->queue; 607 enum nvmet_tcp_recv_state queue_state; 608 struct nvmet_tcp_cmd *queue_cmd; 609 struct nvme_sgl_desc *sgl; 610 u32 len; 611 612 /* Pairs with store_release in nvmet_prepare_receive_pdu() */ 613 queue_state = smp_load_acquire(&queue->rcv_state); 614 queue_cmd = READ_ONCE(queue->cmd); 615 616 if (unlikely(cmd == queue_cmd)) { 617 sgl = &cmd->req.cmd->common.dptr.sgl; 618 len = le32_to_cpu(sgl->length); 619 620 /* 621 * Wait for inline data before processing the response. 622 * Avoid using helpers, this might happen before 623 * nvmet_req_init is completed. 624 */ 625 if (queue_state == NVMET_TCP_RECV_PDU && 626 len && len <= cmd->req.port->inline_data_size && 627 nvme_is_write(cmd->req.cmd)) 628 return; 629 } 630 631 llist_add(&cmd->lentry, &queue->resp_list); 632 queue_work_on(queue_cpu(queue), nvmet_tcp_wq, &cmd->queue->io_work); 633 } 634 635 static void nvmet_tcp_execute_request(struct nvmet_tcp_cmd *cmd) 636 { 637 if (unlikely(cmd->flags & NVMET_TCP_F_INIT_FAILED)) 638 nvmet_tcp_queue_response(&cmd->req); 639 else 640 cmd->req.execute(&cmd->req); 641 } 642 643 static int nvmet_try_send_data_pdu(struct nvmet_tcp_cmd *cmd) 644 { 645 struct msghdr msg = { 646 .msg_flags = MSG_DONTWAIT | MSG_MORE | MSG_SPLICE_PAGES, 647 }; 648 struct bio_vec bvec; 649 u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue); 650 int left = sizeof(*cmd->data_pdu) - cmd->offset + hdgst; 651 int ret; 652 653 bvec_set_virt(&bvec, (void *)cmd->data_pdu + cmd->offset, left); 654 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, left); 655 ret = sock_sendmsg(cmd->queue->sock, &msg); 656 if (ret <= 0) 657 return ret; 658 659 cmd->offset += ret; 660 left -= ret; 661 662 if (left) 663 return -EAGAIN; 664 665 cmd->state = NVMET_TCP_SEND_DATA; 666 cmd->offset = 0; 667 return 1; 668 } 669 670 static int nvmet_try_send_data(struct nvmet_tcp_cmd *cmd, bool last_in_batch) 671 { 672 struct nvmet_tcp_queue *queue = cmd->queue; 673 int ret; 674 675 while (cmd->cur_sg) { 676 struct msghdr msg = { 677 .msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES, 678 }; 679 struct page *page = sg_page(cmd->cur_sg); 680 struct bio_vec bvec; 681 u32 left = cmd->cur_sg->length - cmd->offset; 682 683 if ((!last_in_batch && cmd->queue->send_list_len) || 684 cmd->wbytes_done + left < cmd->req.transfer_len || 685 queue->data_digest || !queue->nvme_sq.sqhd_disabled) 686 msg.msg_flags |= MSG_MORE; 687 688 bvec_set_page(&bvec, page, left, cmd->offset); 689 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, left); 690 ret = sock_sendmsg(cmd->queue->sock, &msg); 691 if (ret <= 0) 692 return ret; 693 694 cmd->offset += ret; 695 cmd->wbytes_done += ret; 696 697 /* Done with sg?*/ 698 if (cmd->offset == cmd->cur_sg->length) { 699 cmd->cur_sg = sg_next(cmd->cur_sg); 700 cmd->offset = 0; 701 } 702 } 703 704 if (queue->data_digest) { 705 cmd->state = NVMET_TCP_SEND_DDGST; 706 cmd->offset = 0; 707 } else { 708 if (queue->nvme_sq.sqhd_disabled) { 709 cmd->queue->snd_cmd = NULL; 710 nvmet_tcp_put_cmd(cmd); 711 } else { 712 nvmet_setup_response_pdu(cmd); 713 } 714 } 715 716 if (queue->nvme_sq.sqhd_disabled) 717 nvmet_tcp_free_cmd_buffers(cmd); 718 719 return 1; 720 721 } 722 723 static int nvmet_try_send_response(struct nvmet_tcp_cmd *cmd, 724 bool last_in_batch) 725 { 726 struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES, }; 727 struct bio_vec bvec; 728 u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue); 729 int left = sizeof(*cmd->rsp_pdu) - cmd->offset + hdgst; 730 int ret; 731 732 if (!last_in_batch && cmd->queue->send_list_len) 733 msg.msg_flags |= MSG_MORE; 734 else 735 msg.msg_flags |= MSG_EOR; 736 737 bvec_set_virt(&bvec, (void *)cmd->rsp_pdu + cmd->offset, left); 738 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, left); 739 ret = sock_sendmsg(cmd->queue->sock, &msg); 740 if (ret <= 0) 741 return ret; 742 cmd->offset += ret; 743 left -= ret; 744 745 if (left) 746 return -EAGAIN; 747 748 nvmet_tcp_free_cmd_buffers(cmd); 749 cmd->queue->snd_cmd = NULL; 750 nvmet_tcp_put_cmd(cmd); 751 return 1; 752 } 753 754 static int nvmet_try_send_r2t(struct nvmet_tcp_cmd *cmd, bool last_in_batch) 755 { 756 struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES, }; 757 struct bio_vec bvec; 758 u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue); 759 int left = sizeof(*cmd->r2t_pdu) - cmd->offset + hdgst; 760 int ret; 761 762 if (!last_in_batch && cmd->queue->send_list_len) 763 msg.msg_flags |= MSG_MORE; 764 else 765 msg.msg_flags |= MSG_EOR; 766 767 bvec_set_virt(&bvec, (void *)cmd->r2t_pdu + cmd->offset, left); 768 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, left); 769 ret = sock_sendmsg(cmd->queue->sock, &msg); 770 if (ret <= 0) 771 return ret; 772 cmd->offset += ret; 773 left -= ret; 774 775 if (left) 776 return -EAGAIN; 777 778 cmd->queue->snd_cmd = NULL; 779 return 1; 780 } 781 782 static int nvmet_try_send_ddgst(struct nvmet_tcp_cmd *cmd, bool last_in_batch) 783 { 784 struct nvmet_tcp_queue *queue = cmd->queue; 785 int left = NVME_TCP_DIGEST_LENGTH - cmd->offset; 786 struct msghdr msg = { .msg_flags = MSG_DONTWAIT }; 787 struct kvec iov = { 788 .iov_base = (u8 *)&cmd->exp_ddgst + cmd->offset, 789 .iov_len = left 790 }; 791 int ret; 792 793 if (!last_in_batch && cmd->queue->send_list_len) 794 msg.msg_flags |= MSG_MORE; 795 else 796 msg.msg_flags |= MSG_EOR; 797 798 ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len); 799 if (unlikely(ret <= 0)) 800 return ret; 801 802 cmd->offset += ret; 803 left -= ret; 804 805 if (left) 806 return -EAGAIN; 807 808 if (queue->nvme_sq.sqhd_disabled) { 809 cmd->queue->snd_cmd = NULL; 810 nvmet_tcp_put_cmd(cmd); 811 } else { 812 nvmet_setup_response_pdu(cmd); 813 } 814 return 1; 815 } 816 817 static int nvmet_tcp_try_send_one(struct nvmet_tcp_queue *queue, 818 bool last_in_batch) 819 { 820 struct nvmet_tcp_cmd *cmd = queue->snd_cmd; 821 int ret = 0; 822 823 if (!cmd || queue->state == NVMET_TCP_Q_DISCONNECTING) { 824 cmd = nvmet_tcp_fetch_cmd(queue); 825 if (unlikely(!cmd)) 826 return 0; 827 } 828 829 if (cmd->state == NVMET_TCP_SEND_DATA_PDU) { 830 ret = nvmet_try_send_data_pdu(cmd); 831 if (ret <= 0) 832 goto done_send; 833 } 834 835 if (cmd->state == NVMET_TCP_SEND_DATA) { 836 ret = nvmet_try_send_data(cmd, last_in_batch); 837 if (ret <= 0) 838 goto done_send; 839 } 840 841 if (cmd->state == NVMET_TCP_SEND_DDGST) { 842 ret = nvmet_try_send_ddgst(cmd, last_in_batch); 843 if (ret <= 0) 844 goto done_send; 845 } 846 847 if (cmd->state == NVMET_TCP_SEND_R2T) { 848 ret = nvmet_try_send_r2t(cmd, last_in_batch); 849 if (ret <= 0) 850 goto done_send; 851 } 852 853 if (cmd->state == NVMET_TCP_SEND_RESPONSE) 854 ret = nvmet_try_send_response(cmd, last_in_batch); 855 856 done_send: 857 if (ret < 0) { 858 if (ret == -EAGAIN) 859 return 0; 860 return ret; 861 } 862 863 return 1; 864 } 865 866 static int nvmet_tcp_try_send(struct nvmet_tcp_queue *queue, 867 int budget, int *sends) 868 { 869 int i, ret = 0; 870 871 for (i = 0; i < budget; i++) { 872 ret = nvmet_tcp_try_send_one(queue, i == budget - 1); 873 if (unlikely(ret < 0)) { 874 nvmet_tcp_socket_error(queue, ret); 875 goto done; 876 } else if (ret == 0) { 877 break; 878 } 879 (*sends)++; 880 } 881 done: 882 return ret; 883 } 884 885 static void nvmet_prepare_receive_pdu(struct nvmet_tcp_queue *queue) 886 { 887 queue->offset = 0; 888 queue->left = sizeof(struct nvme_tcp_hdr); 889 WRITE_ONCE(queue->cmd, NULL); 890 /* Ensure rcv_state is visible only after queue->cmd is set */ 891 smp_store_release(&queue->rcv_state, NVMET_TCP_RECV_PDU); 892 } 893 894 static int nvmet_tcp_handle_icreq(struct nvmet_tcp_queue *queue) 895 { 896 struct nvme_tcp_icreq_pdu *icreq = &queue->pdu.icreq; 897 struct nvme_tcp_icresp_pdu *icresp = &queue->pdu.icresp; 898 struct msghdr msg = {}; 899 struct kvec iov; 900 int ret; 901 902 if (le32_to_cpu(icreq->hdr.plen) != sizeof(struct nvme_tcp_icreq_pdu)) { 903 pr_err("bad nvme-tcp pdu length (%d)\n", 904 le32_to_cpu(icreq->hdr.plen)); 905 return -EPROTO; 906 } 907 908 if (icreq->pfv != NVME_TCP_PFV_1_0) { 909 pr_err("queue %d: bad pfv %d\n", queue->idx, icreq->pfv); 910 return -EPROTO; 911 } 912 913 if (icreq->hpda != 0) { 914 pr_err("queue %d: unsupported hpda %d\n", queue->idx, 915 icreq->hpda); 916 return -EPROTO; 917 } 918 919 queue->hdr_digest = !!(icreq->digest & NVME_TCP_HDR_DIGEST_ENABLE); 920 queue->data_digest = !!(icreq->digest & NVME_TCP_DATA_DIGEST_ENABLE); 921 922 memset(icresp, 0, sizeof(*icresp)); 923 icresp->hdr.type = nvme_tcp_icresp; 924 icresp->hdr.hlen = sizeof(*icresp); 925 icresp->hdr.pdo = 0; 926 icresp->hdr.plen = cpu_to_le32(icresp->hdr.hlen); 927 icresp->pfv = cpu_to_le16(NVME_TCP_PFV_1_0); 928 icresp->maxdata = cpu_to_le32(NVMET_TCP_MAXH2CDATA); 929 icresp->cpda = 0; 930 if (queue->hdr_digest) 931 icresp->digest |= NVME_TCP_HDR_DIGEST_ENABLE; 932 if (queue->data_digest) 933 icresp->digest |= NVME_TCP_DATA_DIGEST_ENABLE; 934 935 iov.iov_base = icresp; 936 iov.iov_len = sizeof(*icresp); 937 ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len); 938 if (ret < 0) { 939 spin_lock_bh(&queue->state_lock); 940 if (queue->state == NVMET_TCP_Q_DISCONNECTING) { 941 spin_unlock_bh(&queue->state_lock); 942 return -ESHUTDOWN; 943 } 944 queue->state = NVMET_TCP_Q_FAILED; 945 spin_unlock_bh(&queue->state_lock); 946 return ret; /* queue removal will cleanup */ 947 } 948 949 spin_lock_bh(&queue->state_lock); 950 if (queue->state == NVMET_TCP_Q_DISCONNECTING) { 951 spin_unlock_bh(&queue->state_lock); 952 /* Tell nvmet_tcp_socket_error() teardown is in progress. */ 953 return -ESHUTDOWN; 954 } 955 queue->state = NVMET_TCP_Q_LIVE; 956 spin_unlock_bh(&queue->state_lock); 957 nvmet_prepare_receive_pdu(queue); 958 return 0; 959 } 960 961 static int nvmet_tcp_handle_req_failure(struct nvmet_tcp_queue *queue, 962 struct nvmet_tcp_cmd *cmd, struct nvmet_req *req) 963 { 964 size_t data_len = le32_to_cpu(req->cmd->common.dptr.sgl.length); 965 int ret; 966 967 /* 968 * This command has not been processed yet, hence we are trying to 969 * figure out if there is still pending data left to receive. If 970 * we don't, we can simply prepare for the next pdu and bail out, 971 * otherwise we will need to prepare a buffer and receive the 972 * stale data before continuing forward. 973 */ 974 if (!nvme_is_write(cmd->req.cmd) || !data_len || 975 data_len > cmd->req.port->inline_data_size) { 976 nvmet_prepare_receive_pdu(queue); 977 return 0; 978 } 979 980 ret = nvmet_tcp_map_data(cmd); 981 if (unlikely(ret)) { 982 pr_err("queue %d: failed to map data\n", queue->idx); 983 return -EPROTO; 984 } 985 986 queue->rcv_state = NVMET_TCP_RECV_DATA; 987 cmd->flags |= NVMET_TCP_F_INIT_FAILED; 988 ret = nvmet_tcp_build_pdu_iovec(cmd); 989 if (unlikely(ret)) 990 pr_err("queue %d: failed to build PDU iovec\n", queue->idx); 991 992 return ret; 993 } 994 995 static int nvmet_tcp_handle_h2c_data_pdu(struct nvmet_tcp_queue *queue) 996 { 997 struct nvme_tcp_data_pdu *data = &queue->pdu.data; 998 struct nvmet_tcp_cmd *cmd; 999 unsigned int exp_data_len; 1000 1001 if (likely(queue->nr_cmds)) { 1002 if (unlikely(data->ttag >= queue->nr_cmds)) { 1003 pr_err("queue %d: received out of bound ttag %u, nr_cmds %u\n", 1004 queue->idx, data->ttag, queue->nr_cmds); 1005 goto err_proto; 1006 } 1007 cmd = &queue->cmds[data->ttag]; 1008 } else { 1009 cmd = &queue->connect; 1010 } 1011 1012 if (le32_to_cpu(data->data_offset) != cmd->rbytes_done) { 1013 pr_err("ttag %u unexpected data offset %u (expected %u)\n", 1014 data->ttag, le32_to_cpu(data->data_offset), 1015 cmd->rbytes_done); 1016 goto err_proto; 1017 } 1018 1019 exp_data_len = le32_to_cpu(data->hdr.plen) - 1020 nvmet_tcp_hdgst_len(queue) - 1021 nvmet_tcp_ddgst_len(queue) - 1022 sizeof(*data); 1023 1024 cmd->pdu_len = le32_to_cpu(data->data_length); 1025 if (unlikely(cmd->pdu_len != exp_data_len || 1026 cmd->pdu_len == 0 || 1027 cmd->pdu_len > NVMET_TCP_MAXH2CDATA)) { 1028 pr_err("H2CData PDU len %u is invalid\n", cmd->pdu_len); 1029 goto err_proto; 1030 } 1031 /* 1032 * Ensure command data structures are initialized. We must check both 1033 * cmd->req.sg and cmd->iov because they can have different NULL states: 1034 * - Uninitialized commands: both NULL 1035 * - READ commands: cmd->req.sg allocated, cmd->iov NULL 1036 * - WRITE commands: both allocated 1037 */ 1038 if (unlikely(!cmd->req.sg || !cmd->iov)) { 1039 pr_err("queue %d: H2CData PDU received for invalid command state (ttag %u)\n", 1040 queue->idx, data->ttag); 1041 goto err_proto; 1042 } 1043 cmd->pdu_recv = 0; 1044 if (unlikely(nvmet_tcp_build_pdu_iovec(cmd))) { 1045 pr_err("queue %d: failed to build PDU iovec\n", queue->idx); 1046 goto err_proto; 1047 } 1048 queue->cmd = cmd; 1049 queue->rcv_state = NVMET_TCP_RECV_DATA; 1050 1051 return 0; 1052 1053 err_proto: 1054 /* FIXME: use proper transport errors */ 1055 return -EPROTO; 1056 } 1057 1058 static int nvmet_tcp_done_recv_pdu(struct nvmet_tcp_queue *queue) 1059 { 1060 struct nvme_tcp_hdr *hdr = &queue->pdu.cmd.hdr; 1061 struct nvme_command *nvme_cmd = &queue->pdu.cmd.cmd; 1062 struct nvmet_req *req; 1063 int ret; 1064 1065 if (unlikely(queue->state == NVMET_TCP_Q_CONNECTING)) { 1066 if (hdr->type != nvme_tcp_icreq) { 1067 pr_err("unexpected pdu type (%d) before icreq\n", 1068 hdr->type); 1069 return -EPROTO; 1070 } 1071 return nvmet_tcp_handle_icreq(queue); 1072 } 1073 1074 if (unlikely(hdr->type == nvme_tcp_icreq)) { 1075 pr_err("queue %d: received icreq pdu in state %d\n", 1076 queue->idx, queue->state); 1077 return -EPROTO; 1078 } 1079 1080 if (hdr->type == nvme_tcp_h2c_data) { 1081 ret = nvmet_tcp_handle_h2c_data_pdu(queue); 1082 if (unlikely(ret)) 1083 return ret; 1084 return 0; 1085 } 1086 1087 queue->cmd = nvmet_tcp_get_cmd(queue); 1088 if (unlikely(!queue->cmd)) { 1089 /* This should never happen */ 1090 pr_err("queue %d: out of commands (%d) send_list_len: %d, opcode: %d", 1091 queue->idx, queue->nr_cmds, queue->send_list_len, 1092 nvme_cmd->common.opcode); 1093 return -ENOMEM; 1094 } 1095 1096 req = &queue->cmd->req; 1097 memcpy(req->cmd, nvme_cmd, sizeof(*nvme_cmd)); 1098 1099 if (unlikely(!nvmet_req_init(req, &queue->nvme_sq, &nvmet_tcp_ops))) { 1100 pr_err("failed cmd %p id %d opcode %d, data_len: %d, status: %04x\n", 1101 req->cmd, req->cmd->common.command_id, 1102 req->cmd->common.opcode, 1103 le32_to_cpu(req->cmd->common.dptr.sgl.length), 1104 le16_to_cpu(req->cqe->status)); 1105 1106 return nvmet_tcp_handle_req_failure(queue, queue->cmd, req); 1107 } 1108 1109 ret = nvmet_tcp_map_data(queue->cmd); 1110 if (unlikely(ret)) { 1111 pr_err("queue %d: failed to map data\n", queue->idx); 1112 if (nvmet_tcp_has_inline_data(queue->cmd)) 1113 return -EPROTO; 1114 1115 nvmet_req_complete(req, ret); 1116 ret = -EAGAIN; 1117 goto out; 1118 } 1119 1120 if (nvmet_tcp_need_data_in(queue->cmd)) { 1121 if (nvmet_tcp_has_inline_data(queue->cmd)) { 1122 queue->rcv_state = NVMET_TCP_RECV_DATA; 1123 ret = nvmet_tcp_build_pdu_iovec(queue->cmd); 1124 if (unlikely(ret)) 1125 pr_err("queue %d: failed to build PDU iovec\n", 1126 queue->idx); 1127 return ret; 1128 } 1129 /* send back R2T */ 1130 nvmet_tcp_queue_response(&queue->cmd->req); 1131 goto out; 1132 } 1133 1134 queue->cmd->req.execute(&queue->cmd->req); 1135 out: 1136 nvmet_prepare_receive_pdu(queue); 1137 return ret; 1138 } 1139 1140 static const u8 nvme_tcp_pdu_sizes[] = { 1141 [nvme_tcp_icreq] = sizeof(struct nvme_tcp_icreq_pdu), 1142 [nvme_tcp_cmd] = sizeof(struct nvme_tcp_cmd_pdu), 1143 [nvme_tcp_h2c_data] = sizeof(struct nvme_tcp_data_pdu), 1144 }; 1145 1146 static inline u8 nvmet_tcp_pdu_size(u8 type) 1147 { 1148 size_t idx = type; 1149 1150 return (idx < ARRAY_SIZE(nvme_tcp_pdu_sizes) && 1151 nvme_tcp_pdu_sizes[idx]) ? 1152 nvme_tcp_pdu_sizes[idx] : 0; 1153 } 1154 1155 static inline bool nvmet_tcp_pdu_valid(u8 type) 1156 { 1157 switch (type) { 1158 case nvme_tcp_icreq: 1159 case nvme_tcp_cmd: 1160 case nvme_tcp_h2c_data: 1161 /* fallthru */ 1162 return true; 1163 } 1164 1165 return false; 1166 } 1167 1168 static int nvmet_tcp_tls_record_ok(struct nvmet_tcp_queue *queue, 1169 struct msghdr *msg, char *cbuf) 1170 { 1171 struct cmsghdr *cmsg = (struct cmsghdr *)cbuf; 1172 u8 ctype, level, description; 1173 int ret = 0; 1174 1175 ctype = tls_get_record_type(queue->sock->sk, cmsg); 1176 switch (ctype) { 1177 case 0: 1178 break; 1179 case TLS_RECORD_TYPE_DATA: 1180 break; 1181 case TLS_RECORD_TYPE_ALERT: 1182 tls_alert_recv(queue->sock->sk, msg, &level, &description); 1183 if (level == TLS_ALERT_LEVEL_FATAL) { 1184 pr_err("queue %d: TLS Alert desc %u\n", 1185 queue->idx, description); 1186 ret = -ENOTCONN; 1187 } else { 1188 pr_warn("queue %d: TLS Alert desc %u\n", 1189 queue->idx, description); 1190 ret = -EAGAIN; 1191 } 1192 break; 1193 default: 1194 /* discard this record type */ 1195 pr_err("queue %d: TLS record %d unhandled\n", 1196 queue->idx, ctype); 1197 ret = -EAGAIN; 1198 break; 1199 } 1200 return ret; 1201 } 1202 1203 static int nvmet_tcp_try_recv_pdu(struct nvmet_tcp_queue *queue) 1204 { 1205 struct nvme_tcp_hdr *hdr = &queue->pdu.cmd.hdr; 1206 int len, ret; 1207 struct kvec iov; 1208 char cbuf[CMSG_LEN(sizeof(char))] = {}; 1209 struct msghdr msg = { .msg_flags = MSG_DONTWAIT }; 1210 1211 recv: 1212 iov.iov_base = (void *)&queue->pdu + queue->offset; 1213 iov.iov_len = queue->left; 1214 if (queue->tls_pskid) { 1215 msg.msg_control = cbuf; 1216 msg.msg_controllen = sizeof(cbuf); 1217 } 1218 len = kernel_recvmsg(queue->sock, &msg, &iov, 1, 1219 iov.iov_len, msg.msg_flags); 1220 if (unlikely(len < 0)) 1221 return len; 1222 if (queue->tls_pskid) { 1223 ret = nvmet_tcp_tls_record_ok(queue, &msg, cbuf); 1224 if (ret < 0) 1225 return ret; 1226 } 1227 1228 queue->offset += len; 1229 queue->left -= len; 1230 if (queue->left) 1231 return -EAGAIN; 1232 1233 if (queue->offset == sizeof(struct nvme_tcp_hdr)) { 1234 u8 hdgst = nvmet_tcp_hdgst_len(queue); 1235 1236 if (unlikely(!nvmet_tcp_pdu_valid(hdr->type))) { 1237 pr_err("unexpected pdu type %d\n", hdr->type); 1238 return -EIO; 1239 } 1240 1241 if (unlikely(hdr->hlen != nvmet_tcp_pdu_size(hdr->type))) { 1242 pr_err("pdu %d bad hlen %d\n", hdr->type, hdr->hlen); 1243 return -EIO; 1244 } 1245 1246 queue->left = hdr->hlen - queue->offset + hdgst; 1247 goto recv; 1248 } 1249 1250 if (queue->hdr_digest && 1251 nvmet_tcp_verify_hdgst(queue, &queue->pdu, hdr->hlen)) 1252 return -EPROTO; 1253 1254 if (queue->data_digest && 1255 nvmet_tcp_check_ddgst(queue, &queue->pdu)) 1256 return -EPROTO; 1257 1258 return nvmet_tcp_done_recv_pdu(queue); 1259 } 1260 1261 static void nvmet_tcp_prep_recv_ddgst(struct nvmet_tcp_cmd *cmd) 1262 { 1263 struct nvmet_tcp_queue *queue = cmd->queue; 1264 1265 nvmet_tcp_calc_ddgst(cmd); 1266 queue->offset = 0; 1267 queue->left = NVME_TCP_DIGEST_LENGTH; 1268 queue->rcv_state = NVMET_TCP_RECV_DDGST; 1269 } 1270 1271 static int nvmet_tcp_try_recv_data(struct nvmet_tcp_queue *queue) 1272 { 1273 struct nvmet_tcp_cmd *cmd = queue->cmd; 1274 int len, ret; 1275 1276 while (msg_data_left(&cmd->recv_msg)) { 1277 len = sock_recvmsg(cmd->queue->sock, &cmd->recv_msg, 1278 cmd->recv_msg.msg_flags); 1279 if (len <= 0) 1280 return len; 1281 if (queue->tls_pskid) { 1282 ret = nvmet_tcp_tls_record_ok(cmd->queue, 1283 &cmd->recv_msg, cmd->recv_cbuf); 1284 if (ret < 0) 1285 return ret; 1286 } 1287 1288 cmd->pdu_recv += len; 1289 cmd->rbytes_done += len; 1290 } 1291 1292 if (queue->data_digest) { 1293 nvmet_tcp_prep_recv_ddgst(cmd); 1294 return 0; 1295 } 1296 1297 if (cmd->rbytes_done == cmd->req.transfer_len) 1298 nvmet_tcp_execute_request(cmd); 1299 1300 nvmet_prepare_receive_pdu(queue); 1301 return 0; 1302 } 1303 1304 static int nvmet_tcp_try_recv_ddgst(struct nvmet_tcp_queue *queue) 1305 { 1306 struct nvmet_tcp_cmd *cmd = queue->cmd; 1307 int ret, len; 1308 char cbuf[CMSG_LEN(sizeof(char))] = {}; 1309 struct msghdr msg = { .msg_flags = MSG_DONTWAIT }; 1310 struct kvec iov = { 1311 .iov_base = (void *)&cmd->recv_ddgst + queue->offset, 1312 .iov_len = queue->left 1313 }; 1314 1315 if (queue->tls_pskid) { 1316 msg.msg_control = cbuf; 1317 msg.msg_controllen = sizeof(cbuf); 1318 } 1319 len = kernel_recvmsg(queue->sock, &msg, &iov, 1, 1320 iov.iov_len, msg.msg_flags); 1321 if (unlikely(len < 0)) 1322 return len; 1323 if (queue->tls_pskid) { 1324 ret = nvmet_tcp_tls_record_ok(queue, &msg, cbuf); 1325 if (ret < 0) 1326 return ret; 1327 } 1328 1329 queue->offset += len; 1330 queue->left -= len; 1331 if (queue->left) 1332 return -EAGAIN; 1333 1334 if (queue->data_digest && cmd->exp_ddgst != cmd->recv_ddgst) { 1335 pr_err("queue %d: cmd %d pdu (%d) data digest error: recv %#x expected %#x\n", 1336 queue->idx, cmd->req.cmd->common.command_id, 1337 queue->pdu.cmd.hdr.type, le32_to_cpu(cmd->recv_ddgst), 1338 le32_to_cpu(cmd->exp_ddgst)); 1339 if (!(cmd->flags & NVMET_TCP_F_INIT_FAILED)) { 1340 cmd->req.cqe->status = NVME_SC_CMD_SEQ_ERROR; 1341 nvmet_req_uninit(&cmd->req); 1342 } 1343 nvmet_tcp_free_cmd_buffers(cmd); 1344 ret = -EPROTO; 1345 goto out; 1346 } 1347 1348 if (cmd->rbytes_done == cmd->req.transfer_len) 1349 nvmet_tcp_execute_request(cmd); 1350 1351 ret = 0; 1352 out: 1353 nvmet_prepare_receive_pdu(queue); 1354 return ret; 1355 } 1356 1357 static int nvmet_tcp_try_recv_one(struct nvmet_tcp_queue *queue) 1358 { 1359 int result = 0; 1360 1361 if (unlikely(queue->rcv_state == NVMET_TCP_RECV_ERR)) 1362 return 0; 1363 1364 if (queue->rcv_state == NVMET_TCP_RECV_PDU) { 1365 result = nvmet_tcp_try_recv_pdu(queue); 1366 if (result != 0) 1367 goto done_recv; 1368 } 1369 1370 if (queue->rcv_state == NVMET_TCP_RECV_DATA) { 1371 result = nvmet_tcp_try_recv_data(queue); 1372 if (result != 0) 1373 goto done_recv; 1374 } 1375 1376 if (queue->rcv_state == NVMET_TCP_RECV_DDGST) { 1377 result = nvmet_tcp_try_recv_ddgst(queue); 1378 if (result != 0) 1379 goto done_recv; 1380 } 1381 1382 done_recv: 1383 if (result < 0) { 1384 if (result == -EAGAIN) 1385 return 0; 1386 return result; 1387 } 1388 return 1; 1389 } 1390 1391 static int nvmet_tcp_try_recv(struct nvmet_tcp_queue *queue, 1392 int budget, int *recvs) 1393 { 1394 int i, ret = 0; 1395 1396 for (i = 0; i < budget; i++) { 1397 ret = nvmet_tcp_try_recv_one(queue); 1398 if (unlikely(ret < 0)) { 1399 nvmet_tcp_socket_error(queue, ret); 1400 goto done; 1401 } else if (ret == 0) { 1402 break; 1403 } 1404 (*recvs)++; 1405 } 1406 done: 1407 return ret; 1408 } 1409 1410 static void nvmet_tcp_release_queue(struct kref *kref) 1411 { 1412 struct nvmet_tcp_queue *queue = 1413 container_of(kref, struct nvmet_tcp_queue, kref); 1414 1415 WARN_ON(queue->state != NVMET_TCP_Q_DISCONNECTING); 1416 queue_work(nvmet_wq, &queue->release_work); 1417 } 1418 1419 static void nvmet_tcp_schedule_release_queue(struct nvmet_tcp_queue *queue) 1420 { 1421 spin_lock_bh(&queue->state_lock); 1422 if (queue->state == NVMET_TCP_Q_TLS_HANDSHAKE) { 1423 /* Socket closed during handshake */ 1424 tls_handshake_cancel(queue->sock->sk); 1425 } 1426 if (queue->state != NVMET_TCP_Q_DISCONNECTING) { 1427 queue->state = NVMET_TCP_Q_DISCONNECTING; 1428 kref_put(&queue->kref, nvmet_tcp_release_queue); 1429 } 1430 spin_unlock_bh(&queue->state_lock); 1431 } 1432 1433 static inline void nvmet_tcp_arm_queue_deadline(struct nvmet_tcp_queue *queue) 1434 { 1435 queue->poll_end = jiffies + usecs_to_jiffies(idle_poll_period_usecs); 1436 } 1437 1438 static bool nvmet_tcp_check_queue_deadline(struct nvmet_tcp_queue *queue, 1439 int ops) 1440 { 1441 if (!idle_poll_period_usecs) 1442 return false; 1443 1444 if (ops) 1445 nvmet_tcp_arm_queue_deadline(queue); 1446 1447 return !time_after(jiffies, queue->poll_end); 1448 } 1449 1450 static void nvmet_tcp_io_work(struct work_struct *w) 1451 { 1452 struct nvmet_tcp_queue *queue = 1453 container_of(w, struct nvmet_tcp_queue, io_work); 1454 bool pending; 1455 int ret, ops = 0; 1456 1457 do { 1458 pending = false; 1459 1460 ret = nvmet_tcp_try_recv(queue, NVMET_TCP_RECV_BUDGET, &ops); 1461 if (ret > 0) 1462 pending = true; 1463 else if (ret < 0) 1464 return; 1465 1466 ret = nvmet_tcp_try_send(queue, NVMET_TCP_SEND_BUDGET, &ops); 1467 if (ret > 0) 1468 pending = true; 1469 else if (ret < 0) 1470 return; 1471 1472 } while (pending && ops < NVMET_TCP_IO_WORK_BUDGET); 1473 1474 /* 1475 * Requeue the worker if idle deadline period is in progress or any 1476 * ops activity was recorded during the do-while loop above. 1477 */ 1478 if (nvmet_tcp_check_queue_deadline(queue, ops) || pending) 1479 queue_work_on(queue_cpu(queue), nvmet_tcp_wq, &queue->io_work); 1480 } 1481 1482 static int nvmet_tcp_alloc_cmd(struct nvmet_tcp_queue *queue, 1483 struct nvmet_tcp_cmd *c) 1484 { 1485 u8 hdgst = nvmet_tcp_hdgst_len(queue); 1486 1487 c->queue = queue; 1488 c->req.port = queue->port->nport; 1489 1490 c->cmd_pdu = page_frag_alloc(&queue->pf_cache, 1491 sizeof(*c->cmd_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO); 1492 if (!c->cmd_pdu) 1493 return -ENOMEM; 1494 c->req.cmd = &c->cmd_pdu->cmd; 1495 1496 c->rsp_pdu = page_frag_alloc(&queue->pf_cache, 1497 sizeof(*c->rsp_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO); 1498 if (!c->rsp_pdu) 1499 goto out_free_cmd; 1500 c->req.cqe = &c->rsp_pdu->cqe; 1501 1502 c->data_pdu = page_frag_alloc(&queue->pf_cache, 1503 sizeof(*c->data_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO); 1504 if (!c->data_pdu) 1505 goto out_free_rsp; 1506 1507 c->r2t_pdu = page_frag_alloc(&queue->pf_cache, 1508 sizeof(*c->r2t_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO); 1509 if (!c->r2t_pdu) 1510 goto out_free_data; 1511 1512 if (queue->state == NVMET_TCP_Q_TLS_HANDSHAKE) { 1513 c->recv_msg.msg_control = c->recv_cbuf; 1514 c->recv_msg.msg_controllen = sizeof(c->recv_cbuf); 1515 } 1516 c->recv_msg.msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL; 1517 1518 list_add_tail(&c->entry, &queue->free_list); 1519 1520 return 0; 1521 out_free_data: 1522 page_frag_free(c->data_pdu); 1523 out_free_rsp: 1524 page_frag_free(c->rsp_pdu); 1525 out_free_cmd: 1526 page_frag_free(c->cmd_pdu); 1527 return -ENOMEM; 1528 } 1529 1530 static void nvmet_tcp_free_cmd(struct nvmet_tcp_cmd *c) 1531 { 1532 page_frag_free(c->r2t_pdu); 1533 page_frag_free(c->data_pdu); 1534 page_frag_free(c->rsp_pdu); 1535 page_frag_free(c->cmd_pdu); 1536 } 1537 1538 static int nvmet_tcp_alloc_cmds(struct nvmet_tcp_queue *queue) 1539 { 1540 struct nvmet_tcp_cmd *cmds; 1541 int i, ret = -EINVAL, nr_cmds = queue->nr_cmds; 1542 1543 cmds = kvzalloc_objs(struct nvmet_tcp_cmd, nr_cmds); 1544 if (!cmds) 1545 goto out; 1546 1547 for (i = 0; i < nr_cmds; i++) { 1548 ret = nvmet_tcp_alloc_cmd(queue, cmds + i); 1549 if (ret) 1550 goto out_free; 1551 } 1552 1553 queue->cmds = cmds; 1554 1555 return 0; 1556 out_free: 1557 while (--i >= 0) 1558 nvmet_tcp_free_cmd(cmds + i); 1559 kvfree(cmds); 1560 out: 1561 return ret; 1562 } 1563 1564 static void nvmet_tcp_free_cmds(struct nvmet_tcp_queue *queue) 1565 { 1566 struct nvmet_tcp_cmd *cmds = queue->cmds; 1567 int i; 1568 1569 for (i = 0; i < queue->nr_cmds; i++) 1570 nvmet_tcp_free_cmd(cmds + i); 1571 1572 nvmet_tcp_free_cmd(&queue->connect); 1573 kvfree(cmds); 1574 } 1575 1576 static void nvmet_tcp_restore_socket_callbacks(struct nvmet_tcp_queue *queue) 1577 { 1578 struct socket *sock = queue->sock; 1579 1580 if (!queue->state_change) 1581 return; 1582 1583 write_lock_bh(&sock->sk->sk_callback_lock); 1584 sock->sk->sk_data_ready = queue->data_ready; 1585 sock->sk->sk_state_change = queue->state_change; 1586 sock->sk->sk_write_space = queue->write_space; 1587 sock->sk->sk_user_data = NULL; 1588 write_unlock_bh(&sock->sk->sk_callback_lock); 1589 } 1590 1591 static void nvmet_tcp_uninit_data_in_cmds(struct nvmet_tcp_queue *queue) 1592 { 1593 struct nvmet_tcp_cmd *cmd = queue->cmds; 1594 int i; 1595 1596 for (i = 0; i < queue->nr_cmds; i++, cmd++) { 1597 if (nvmet_tcp_need_data_in(cmd)) 1598 nvmet_req_uninit(&cmd->req); 1599 } 1600 1601 if (!queue->nr_cmds && nvmet_tcp_need_data_in(&queue->connect)) { 1602 /* failed in connect */ 1603 nvmet_req_uninit(&queue->connect.req); 1604 } 1605 } 1606 1607 static void nvmet_tcp_free_cmd_data_in_buffers(struct nvmet_tcp_queue *queue) 1608 { 1609 struct nvmet_tcp_cmd *cmd = queue->cmds; 1610 int i; 1611 1612 for (i = 0; i < queue->nr_cmds; i++, cmd++) 1613 nvmet_tcp_free_cmd_buffers(cmd); 1614 nvmet_tcp_free_cmd_buffers(&queue->connect); 1615 } 1616 1617 static void nvmet_tcp_release_queue_work(struct work_struct *w) 1618 { 1619 struct nvmet_tcp_queue *queue = 1620 container_of(w, struct nvmet_tcp_queue, release_work); 1621 1622 mutex_lock(&nvmet_tcp_queue_mutex); 1623 list_del_init(&queue->queue_list); 1624 mutex_unlock(&nvmet_tcp_queue_mutex); 1625 1626 nvmet_tcp_restore_socket_callbacks(queue); 1627 cancel_delayed_work_sync(&queue->tls_handshake_tmo_work); 1628 cancel_work_sync(&queue->io_work); 1629 /* stop accepting incoming data */ 1630 queue->rcv_state = NVMET_TCP_RECV_ERR; 1631 1632 nvmet_sq_put_tls_key(&queue->nvme_sq); 1633 nvmet_tcp_uninit_data_in_cmds(queue); 1634 nvmet_sq_destroy(&queue->nvme_sq); 1635 nvmet_cq_put(&queue->nvme_cq); 1636 cancel_work_sync(&queue->io_work); 1637 nvmet_tcp_free_cmd_data_in_buffers(queue); 1638 /* ->sock will be released by fput() */ 1639 fput(queue->sock->file); 1640 nvmet_tcp_free_cmds(queue); 1641 ida_free(&nvmet_tcp_queue_ida, queue->idx); 1642 page_frag_cache_drain(&queue->pf_cache); 1643 kfree(queue); 1644 } 1645 1646 static void nvmet_tcp_data_ready(struct sock *sk) 1647 { 1648 struct nvmet_tcp_queue *queue; 1649 1650 trace_sk_data_ready(sk); 1651 1652 read_lock_bh(&sk->sk_callback_lock); 1653 queue = sk->sk_user_data; 1654 if (likely(queue)) { 1655 if (queue->data_ready) 1656 queue->data_ready(sk); 1657 if (queue->state != NVMET_TCP_Q_TLS_HANDSHAKE) 1658 queue_work_on(queue_cpu(queue), nvmet_tcp_wq, 1659 &queue->io_work); 1660 } 1661 read_unlock_bh(&sk->sk_callback_lock); 1662 } 1663 1664 static void nvmet_tcp_write_space(struct sock *sk) 1665 { 1666 struct nvmet_tcp_queue *queue; 1667 1668 read_lock_bh(&sk->sk_callback_lock); 1669 queue = sk->sk_user_data; 1670 if (unlikely(!queue)) 1671 goto out; 1672 1673 if (unlikely(queue->state == NVMET_TCP_Q_CONNECTING)) { 1674 queue->write_space(sk); 1675 goto out; 1676 } 1677 1678 if (sk_stream_is_writeable(sk)) { 1679 clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 1680 queue_work_on(queue_cpu(queue), nvmet_tcp_wq, &queue->io_work); 1681 } 1682 out: 1683 read_unlock_bh(&sk->sk_callback_lock); 1684 } 1685 1686 static void nvmet_tcp_state_change(struct sock *sk) 1687 { 1688 struct nvmet_tcp_queue *queue; 1689 1690 read_lock_bh(&sk->sk_callback_lock); 1691 queue = sk->sk_user_data; 1692 if (!queue) 1693 goto done; 1694 1695 switch (sk->sk_state) { 1696 case TCP_FIN_WAIT2: 1697 case TCP_LAST_ACK: 1698 case TCP_CLOSING: 1699 break; 1700 case TCP_FIN_WAIT1: 1701 case TCP_CLOSE_WAIT: 1702 case TCP_CLOSE: 1703 /* FALLTHRU */ 1704 nvmet_tcp_schedule_release_queue(queue); 1705 break; 1706 default: 1707 pr_warn("queue %d unhandled state %d\n", 1708 queue->idx, sk->sk_state); 1709 } 1710 done: 1711 read_unlock_bh(&sk->sk_callback_lock); 1712 } 1713 1714 static int nvmet_tcp_set_queue_sock(struct nvmet_tcp_queue *queue) 1715 { 1716 struct socket *sock = queue->sock; 1717 struct inet_sock *inet = inet_sk(sock->sk); 1718 int ret; 1719 1720 ret = kernel_getsockname(sock, 1721 (struct sockaddr *)&queue->sockaddr); 1722 if (ret < 0) 1723 return ret; 1724 1725 ret = kernel_getpeername(sock, 1726 (struct sockaddr *)&queue->sockaddr_peer); 1727 if (ret < 0) 1728 return ret; 1729 1730 /* 1731 * Cleanup whatever is sitting in the TCP transmit queue on socket 1732 * close. This is done to prevent stale data from being sent should 1733 * the network connection be restored before TCP times out. 1734 */ 1735 sock_no_linger(sock->sk); 1736 1737 if (so_priority > 0) 1738 sock_set_priority(sock->sk, so_priority); 1739 1740 /* Set socket type of service */ 1741 if (inet->rcv_tos > 0) 1742 ip_sock_set_tos(sock->sk, inet->rcv_tos); 1743 1744 ret = 0; 1745 write_lock_bh(&sock->sk->sk_callback_lock); 1746 if (sock->sk->sk_state != TCP_ESTABLISHED) { 1747 /* 1748 * If the socket is already closing, don't even start 1749 * consuming it 1750 */ 1751 ret = -ENOTCONN; 1752 } else { 1753 sock->sk->sk_user_data = queue; 1754 queue->data_ready = sock->sk->sk_data_ready; 1755 sock->sk->sk_data_ready = nvmet_tcp_data_ready; 1756 queue->state_change = sock->sk->sk_state_change; 1757 sock->sk->sk_state_change = nvmet_tcp_state_change; 1758 queue->write_space = sock->sk->sk_write_space; 1759 sock->sk->sk_write_space = nvmet_tcp_write_space; 1760 if (idle_poll_period_usecs) 1761 nvmet_tcp_arm_queue_deadline(queue); 1762 queue_work_on(queue_cpu(queue), nvmet_tcp_wq, &queue->io_work); 1763 } 1764 write_unlock_bh(&sock->sk->sk_callback_lock); 1765 1766 return ret; 1767 } 1768 1769 #ifdef CONFIG_NVME_TARGET_TCP_TLS 1770 static int nvmet_tcp_try_peek_pdu(struct nvmet_tcp_queue *queue) 1771 { 1772 struct nvme_tcp_hdr *hdr = &queue->pdu.cmd.hdr; 1773 int len, ret; 1774 struct kvec iov = { 1775 .iov_base = (u8 *)&queue->pdu + queue->offset, 1776 .iov_len = sizeof(struct nvme_tcp_hdr), 1777 }; 1778 char cbuf[CMSG_LEN(sizeof(char))] = {}; 1779 struct msghdr msg = { 1780 .msg_control = cbuf, 1781 .msg_controllen = sizeof(cbuf), 1782 .msg_flags = MSG_PEEK, 1783 }; 1784 1785 if (nvmet_port_secure_channel_required(queue->port->nport)) 1786 return 0; 1787 1788 len = kernel_recvmsg(queue->sock, &msg, &iov, 1, 1789 iov.iov_len, msg.msg_flags); 1790 if (unlikely(len < 0)) { 1791 pr_debug("queue %d: peek error %d\n", 1792 queue->idx, len); 1793 return len; 1794 } 1795 1796 ret = nvmet_tcp_tls_record_ok(queue, &msg, cbuf); 1797 if (ret < 0) 1798 return ret; 1799 1800 if (len < sizeof(struct nvme_tcp_hdr)) { 1801 pr_debug("queue %d: short read, %d bytes missing\n", 1802 queue->idx, (int)iov.iov_len - len); 1803 return -EAGAIN; 1804 } 1805 pr_debug("queue %d: hdr type %d hlen %d plen %d size %d\n", 1806 queue->idx, hdr->type, hdr->hlen, hdr->plen, 1807 (int)sizeof(struct nvme_tcp_icreq_pdu)); 1808 if (hdr->type == nvme_tcp_icreq && 1809 hdr->hlen == sizeof(struct nvme_tcp_icreq_pdu) && 1810 hdr->plen == cpu_to_le32(sizeof(struct nvme_tcp_icreq_pdu))) { 1811 pr_debug("queue %d: icreq detected\n", 1812 queue->idx); 1813 return len; 1814 } 1815 return 0; 1816 } 1817 1818 static int nvmet_tcp_tls_key_lookup(struct nvmet_tcp_queue *queue, 1819 key_serial_t peerid) 1820 { 1821 struct key *tls_key = nvme_tls_key_lookup(peerid); 1822 int status = 0; 1823 1824 if (IS_ERR(tls_key)) { 1825 pr_warn("%s: queue %d failed to lookup key %x\n", 1826 __func__, queue->idx, peerid); 1827 spin_lock_bh(&queue->state_lock); 1828 queue->state = NVMET_TCP_Q_FAILED; 1829 spin_unlock_bh(&queue->state_lock); 1830 status = PTR_ERR(tls_key); 1831 } else { 1832 pr_debug("%s: queue %d using TLS PSK %x\n", 1833 __func__, queue->idx, peerid); 1834 queue->nvme_sq.tls_key = tls_key; 1835 } 1836 return status; 1837 } 1838 1839 static void nvmet_tcp_tls_handshake_done(void *data, int status, 1840 key_serial_t peerid) 1841 { 1842 struct nvmet_tcp_queue *queue = data; 1843 1844 pr_debug("queue %d: TLS handshake done, key %x, status %d\n", 1845 queue->idx, peerid, status); 1846 spin_lock_bh(&queue->state_lock); 1847 if (WARN_ON(queue->state != NVMET_TCP_Q_TLS_HANDSHAKE)) { 1848 spin_unlock_bh(&queue->state_lock); 1849 return; 1850 } 1851 if (!status) { 1852 queue->tls_pskid = peerid; 1853 queue->state = NVMET_TCP_Q_CONNECTING; 1854 } else 1855 queue->state = NVMET_TCP_Q_FAILED; 1856 spin_unlock_bh(&queue->state_lock); 1857 1858 cancel_delayed_work_sync(&queue->tls_handshake_tmo_work); 1859 1860 if (!status) 1861 status = nvmet_tcp_tls_key_lookup(queue, peerid); 1862 1863 if (!status) 1864 status = nvmet_tcp_set_queue_sock(queue); 1865 1866 if (status) 1867 nvmet_tcp_schedule_release_queue(queue); 1868 kref_put(&queue->kref, nvmet_tcp_release_queue); 1869 } 1870 1871 static void nvmet_tcp_tls_handshake_timeout(struct work_struct *w) 1872 { 1873 struct nvmet_tcp_queue *queue = container_of(to_delayed_work(w), 1874 struct nvmet_tcp_queue, tls_handshake_tmo_work); 1875 1876 pr_warn("queue %d: TLS handshake timeout\n", queue->idx); 1877 /* 1878 * If tls_handshake_cancel() fails we've lost the race with 1879 * nvmet_tcp_tls_handshake_done() */ 1880 if (!tls_handshake_cancel(queue->sock->sk)) 1881 return; 1882 spin_lock_bh(&queue->state_lock); 1883 if (WARN_ON(queue->state != NVMET_TCP_Q_TLS_HANDSHAKE)) { 1884 spin_unlock_bh(&queue->state_lock); 1885 return; 1886 } 1887 queue->state = NVMET_TCP_Q_FAILED; 1888 spin_unlock_bh(&queue->state_lock); 1889 nvmet_tcp_schedule_release_queue(queue); 1890 kref_put(&queue->kref, nvmet_tcp_release_queue); 1891 } 1892 1893 static int nvmet_tcp_tls_handshake(struct nvmet_tcp_queue *queue) 1894 { 1895 int ret = -EOPNOTSUPP; 1896 struct tls_handshake_args args; 1897 1898 if (queue->state != NVMET_TCP_Q_TLS_HANDSHAKE) { 1899 pr_warn("cannot start TLS in state %d\n", queue->state); 1900 return -EINVAL; 1901 } 1902 1903 kref_get(&queue->kref); 1904 pr_debug("queue %d: TLS ServerHello\n", queue->idx); 1905 memset(&args, 0, sizeof(args)); 1906 args.ta_sock = queue->sock; 1907 args.ta_done = nvmet_tcp_tls_handshake_done; 1908 args.ta_data = queue; 1909 args.ta_keyring = key_serial(queue->port->nport->keyring); 1910 args.ta_timeout_ms = tls_handshake_timeout * 1000; 1911 1912 ret = tls_server_hello_psk(&args, GFP_KERNEL); 1913 if (ret) { 1914 kref_put(&queue->kref, nvmet_tcp_release_queue); 1915 pr_err("failed to start TLS, err=%d\n", ret); 1916 } else { 1917 queue_delayed_work(nvmet_wq, &queue->tls_handshake_tmo_work, 1918 tls_handshake_timeout * HZ); 1919 } 1920 return ret; 1921 } 1922 #else 1923 static void nvmet_tcp_tls_handshake_timeout(struct work_struct *w) {} 1924 #endif 1925 1926 static void nvmet_tcp_alloc_queue(struct nvmet_tcp_port *port, 1927 struct socket *newsock) 1928 { 1929 struct nvmet_tcp_queue *queue; 1930 struct file *sock_file = NULL; 1931 int ret; 1932 1933 queue = kzalloc_obj(*queue); 1934 if (!queue) { 1935 ret = -ENOMEM; 1936 goto out_release; 1937 } 1938 1939 INIT_WORK(&queue->release_work, nvmet_tcp_release_queue_work); 1940 INIT_WORK(&queue->io_work, nvmet_tcp_io_work); 1941 kref_init(&queue->kref); 1942 queue->sock = newsock; 1943 queue->port = port; 1944 queue->nr_cmds = 0; 1945 spin_lock_init(&queue->state_lock); 1946 if (queue->port->nport->disc_addr.tsas.tcp.sectype == 1947 NVMF_TCP_SECTYPE_TLS13) 1948 queue->state = NVMET_TCP_Q_TLS_HANDSHAKE; 1949 else 1950 queue->state = NVMET_TCP_Q_CONNECTING; 1951 INIT_LIST_HEAD(&queue->free_list); 1952 init_llist_head(&queue->resp_list); 1953 INIT_LIST_HEAD(&queue->resp_send_list); 1954 1955 sock_file = sock_alloc_file(queue->sock, O_CLOEXEC, NULL); 1956 if (IS_ERR(sock_file)) { 1957 ret = PTR_ERR(sock_file); 1958 goto out_free_queue; 1959 } 1960 1961 queue->idx = ida_alloc(&nvmet_tcp_queue_ida, GFP_KERNEL); 1962 if (queue->idx < 0) { 1963 ret = queue->idx; 1964 goto out_sock; 1965 } 1966 1967 ret = nvmet_tcp_alloc_cmd(queue, &queue->connect); 1968 if (ret) 1969 goto out_ida_remove; 1970 1971 nvmet_cq_init(&queue->nvme_cq); 1972 ret = nvmet_sq_init(&queue->nvme_sq, &queue->nvme_cq); 1973 if (ret) 1974 goto out_free_connect; 1975 1976 nvmet_prepare_receive_pdu(queue); 1977 1978 mutex_lock(&nvmet_tcp_queue_mutex); 1979 list_add_tail(&queue->queue_list, &nvmet_tcp_queue_list); 1980 mutex_unlock(&nvmet_tcp_queue_mutex); 1981 1982 INIT_DELAYED_WORK(&queue->tls_handshake_tmo_work, 1983 nvmet_tcp_tls_handshake_timeout); 1984 #ifdef CONFIG_NVME_TARGET_TCP_TLS 1985 if (queue->state == NVMET_TCP_Q_TLS_HANDSHAKE) { 1986 struct sock *sk = queue->sock->sk; 1987 1988 /* Restore the default callbacks before starting upcall */ 1989 write_lock_bh(&sk->sk_callback_lock); 1990 sk->sk_user_data = NULL; 1991 sk->sk_data_ready = port->data_ready; 1992 write_unlock_bh(&sk->sk_callback_lock); 1993 if (!nvmet_tcp_try_peek_pdu(queue)) { 1994 if (!nvmet_tcp_tls_handshake(queue)) 1995 return; 1996 /* TLS handshake failed, terminate the connection */ 1997 goto out_destroy_sq; 1998 } 1999 /* Not a TLS connection, continue with normal processing */ 2000 queue->state = NVMET_TCP_Q_CONNECTING; 2001 } 2002 #endif 2003 2004 ret = nvmet_tcp_set_queue_sock(queue); 2005 if (ret) 2006 goto out_destroy_sq; 2007 2008 return; 2009 out_destroy_sq: 2010 mutex_lock(&nvmet_tcp_queue_mutex); 2011 list_del_init(&queue->queue_list); 2012 mutex_unlock(&nvmet_tcp_queue_mutex); 2013 nvmet_sq_destroy(&queue->nvme_sq); 2014 out_free_connect: 2015 nvmet_cq_put(&queue->nvme_cq); 2016 nvmet_tcp_free_cmd(&queue->connect); 2017 out_ida_remove: 2018 ida_free(&nvmet_tcp_queue_ida, queue->idx); 2019 /* 2020 * Drain the page fragment cache if any allocations were done. 2021 * The first allocation using pf_cache is nvmet_tcp_alloc_cmd() 2022 * for queue->connect after ida_alloc(). 2023 */ 2024 page_frag_cache_drain(&queue->pf_cache); 2025 out_sock: 2026 fput(queue->sock->file); 2027 out_free_queue: 2028 kfree(queue); 2029 out_release: 2030 pr_err("failed to allocate queue, error %d\n", ret); 2031 if (!sock_file) 2032 sock_release(newsock); 2033 } 2034 2035 static void nvmet_tcp_accept_work(struct work_struct *w) 2036 { 2037 struct nvmet_tcp_port *port = 2038 container_of(w, struct nvmet_tcp_port, accept_work); 2039 struct socket *newsock; 2040 int ret; 2041 2042 while (true) { 2043 ret = kernel_accept(port->sock, &newsock, O_NONBLOCK); 2044 if (ret < 0) { 2045 if (ret != -EAGAIN) 2046 pr_warn("failed to accept err=%d\n", ret); 2047 return; 2048 } 2049 nvmet_tcp_alloc_queue(port, newsock); 2050 } 2051 } 2052 2053 static void nvmet_tcp_listen_data_ready(struct sock *sk) 2054 { 2055 struct nvmet_tcp_port *port; 2056 2057 trace_sk_data_ready(sk); 2058 2059 if (sk->sk_state != TCP_LISTEN) 2060 return; 2061 2062 read_lock_bh(&sk->sk_callback_lock); 2063 port = sk->sk_user_data; 2064 if (port) 2065 queue_work(nvmet_wq, &port->accept_work); 2066 read_unlock_bh(&sk->sk_callback_lock); 2067 } 2068 2069 static int nvmet_tcp_add_port(struct nvmet_port *nport) 2070 { 2071 struct nvmet_tcp_port *port; 2072 __kernel_sa_family_t af; 2073 int ret; 2074 2075 port = kzalloc_obj(*port); 2076 if (!port) 2077 return -ENOMEM; 2078 2079 switch (nport->disc_addr.adrfam) { 2080 case NVMF_ADDR_FAMILY_IP4: 2081 af = AF_INET; 2082 break; 2083 case NVMF_ADDR_FAMILY_IP6: 2084 af = AF_INET6; 2085 break; 2086 default: 2087 pr_err("address family %d not supported\n", 2088 nport->disc_addr.adrfam); 2089 ret = -EINVAL; 2090 goto err_port; 2091 } 2092 2093 ret = inet_pton_with_scope(&init_net, af, nport->disc_addr.traddr, 2094 nport->disc_addr.trsvcid, &port->addr); 2095 if (ret) { 2096 pr_err("malformed ip/port passed: %s:%s\n", 2097 nport->disc_addr.traddr, nport->disc_addr.trsvcid); 2098 goto err_port; 2099 } 2100 2101 port->nport = nport; 2102 INIT_WORK(&port->accept_work, nvmet_tcp_accept_work); 2103 if (port->nport->inline_data_size < 0) 2104 port->nport->inline_data_size = NVMET_TCP_DEF_INLINE_DATA_SIZE; 2105 2106 ret = sock_create(port->addr.ss_family, SOCK_STREAM, 2107 IPPROTO_TCP, &port->sock); 2108 if (ret) { 2109 pr_err("failed to create a socket\n"); 2110 goto err_port; 2111 } 2112 2113 port->sock->sk->sk_user_data = port; 2114 port->data_ready = port->sock->sk->sk_data_ready; 2115 port->sock->sk->sk_data_ready = nvmet_tcp_listen_data_ready; 2116 sock_set_reuseaddr(port->sock->sk); 2117 tcp_sock_set_nodelay(port->sock->sk); 2118 if (so_priority > 0) 2119 sock_set_priority(port->sock->sk, so_priority); 2120 2121 ret = kernel_bind(port->sock, (struct sockaddr_unsized *)&port->addr, 2122 sizeof(port->addr)); 2123 if (ret) { 2124 pr_err("failed to bind port socket %d\n", ret); 2125 goto err_sock; 2126 } 2127 2128 ret = kernel_listen(port->sock, NVMET_TCP_BACKLOG); 2129 if (ret) { 2130 pr_err("failed to listen %d on port sock\n", ret); 2131 goto err_sock; 2132 } 2133 2134 nport->priv = port; 2135 pr_info("enabling port %d (%pISpc)\n", 2136 le16_to_cpu(nport->disc_addr.portid), &port->addr); 2137 2138 return 0; 2139 2140 err_sock: 2141 sock_release(port->sock); 2142 err_port: 2143 kfree(port); 2144 return ret; 2145 } 2146 2147 static void nvmet_tcp_destroy_port_queues(struct nvmet_tcp_port *port) 2148 { 2149 struct nvmet_tcp_queue *queue; 2150 2151 mutex_lock(&nvmet_tcp_queue_mutex); 2152 list_for_each_entry(queue, &nvmet_tcp_queue_list, queue_list) 2153 if (queue->port == port) 2154 kernel_sock_shutdown(queue->sock, SHUT_RDWR); 2155 mutex_unlock(&nvmet_tcp_queue_mutex); 2156 } 2157 2158 static void nvmet_tcp_remove_port(struct nvmet_port *nport) 2159 { 2160 struct nvmet_tcp_port *port = nport->priv; 2161 2162 write_lock_bh(&port->sock->sk->sk_callback_lock); 2163 port->sock->sk->sk_data_ready = port->data_ready; 2164 port->sock->sk->sk_user_data = NULL; 2165 write_unlock_bh(&port->sock->sk->sk_callback_lock); 2166 cancel_work_sync(&port->accept_work); 2167 /* 2168 * Destroy the remaining queues, which are not belong to any 2169 * controller yet. 2170 */ 2171 nvmet_tcp_destroy_port_queues(port); 2172 2173 sock_release(port->sock); 2174 kfree(port); 2175 } 2176 2177 static void nvmet_tcp_delete_ctrl(struct nvmet_ctrl *ctrl) 2178 { 2179 struct nvmet_tcp_queue *queue; 2180 2181 mutex_lock(&nvmet_tcp_queue_mutex); 2182 list_for_each_entry(queue, &nvmet_tcp_queue_list, queue_list) 2183 if (queue->nvme_sq.ctrl == ctrl) 2184 kernel_sock_shutdown(queue->sock, SHUT_RDWR); 2185 mutex_unlock(&nvmet_tcp_queue_mutex); 2186 } 2187 2188 static u16 nvmet_tcp_install_queue(struct nvmet_sq *sq) 2189 { 2190 struct nvmet_tcp_queue *queue = 2191 container_of(sq, struct nvmet_tcp_queue, nvme_sq); 2192 2193 if (sq->qid == 0) { 2194 struct nvmet_tcp_queue *q; 2195 int pending = 0; 2196 2197 /* Check for pending controller teardown */ 2198 mutex_lock(&nvmet_tcp_queue_mutex); 2199 list_for_each_entry(q, &nvmet_tcp_queue_list, queue_list) { 2200 if (q->nvme_sq.ctrl == sq->ctrl && 2201 q->state == NVMET_TCP_Q_DISCONNECTING) 2202 pending++; 2203 } 2204 mutex_unlock(&nvmet_tcp_queue_mutex); 2205 if (pending > NVMET_TCP_BACKLOG) 2206 return NVME_SC_CONNECT_CTRL_BUSY; 2207 } 2208 2209 queue->nr_cmds = sq->size * 2; 2210 if (nvmet_tcp_alloc_cmds(queue)) { 2211 queue->nr_cmds = 0; 2212 return NVME_SC_INTERNAL; 2213 } 2214 return 0; 2215 } 2216 2217 static void nvmet_tcp_disc_port_addr(struct nvmet_req *req, 2218 struct nvmet_port *nport, char *traddr) 2219 { 2220 struct nvmet_tcp_port *port = nport->priv; 2221 2222 if (inet_addr_is_any(&port->addr)) { 2223 struct nvmet_tcp_cmd *cmd = 2224 container_of(req, struct nvmet_tcp_cmd, req); 2225 struct nvmet_tcp_queue *queue = cmd->queue; 2226 2227 sprintf(traddr, "%pISc", (struct sockaddr *)&queue->sockaddr); 2228 } else { 2229 memcpy(traddr, nport->disc_addr.traddr, NVMF_TRADDR_SIZE); 2230 } 2231 } 2232 2233 static ssize_t nvmet_tcp_host_port_addr(struct nvmet_ctrl *ctrl, 2234 char *traddr, size_t traddr_len) 2235 { 2236 struct nvmet_sq *sq = ctrl->sqs[0]; 2237 struct nvmet_tcp_queue *queue = 2238 container_of(sq, struct nvmet_tcp_queue, nvme_sq); 2239 2240 if (queue->sockaddr_peer.ss_family == AF_UNSPEC) 2241 return -EINVAL; 2242 return snprintf(traddr, traddr_len, "%pISc", 2243 (struct sockaddr *)&queue->sockaddr_peer); 2244 } 2245 2246 static const struct nvmet_fabrics_ops nvmet_tcp_ops = { 2247 .owner = THIS_MODULE, 2248 .type = NVMF_TRTYPE_TCP, 2249 .msdbd = 1, 2250 .add_port = nvmet_tcp_add_port, 2251 .remove_port = nvmet_tcp_remove_port, 2252 .queue_response = nvmet_tcp_queue_response, 2253 .delete_ctrl = nvmet_tcp_delete_ctrl, 2254 .install_queue = nvmet_tcp_install_queue, 2255 .disc_traddr = nvmet_tcp_disc_port_addr, 2256 .host_traddr = nvmet_tcp_host_port_addr, 2257 }; 2258 2259 static int __init nvmet_tcp_init(void) 2260 { 2261 int ret; 2262 2263 nvmet_tcp_wq = alloc_workqueue("nvmet_tcp_wq", 2264 WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_PERCPU, 0); 2265 if (!nvmet_tcp_wq) 2266 return -ENOMEM; 2267 2268 ret = nvmet_register_transport(&nvmet_tcp_ops); 2269 if (ret) 2270 goto err; 2271 2272 return 0; 2273 err: 2274 destroy_workqueue(nvmet_tcp_wq); 2275 return ret; 2276 } 2277 2278 static void __exit nvmet_tcp_exit(void) 2279 { 2280 struct nvmet_tcp_queue *queue; 2281 2282 nvmet_unregister_transport(&nvmet_tcp_ops); 2283 2284 flush_workqueue(nvmet_wq); 2285 mutex_lock(&nvmet_tcp_queue_mutex); 2286 list_for_each_entry(queue, &nvmet_tcp_queue_list, queue_list) 2287 kernel_sock_shutdown(queue->sock, SHUT_RDWR); 2288 mutex_unlock(&nvmet_tcp_queue_mutex); 2289 flush_workqueue(nvmet_wq); 2290 2291 destroy_workqueue(nvmet_tcp_wq); 2292 ida_destroy(&nvmet_tcp_queue_ida); 2293 } 2294 2295 module_init(nvmet_tcp_init); 2296 module_exit(nvmet_tcp_exit); 2297 2298 MODULE_DESCRIPTION("NVMe target TCP transport driver"); 2299 MODULE_LICENSE("GPL v2"); 2300 MODULE_ALIAS("nvmet-transport-3"); /* 3 == NVMF_TRTYPE_TCP */ 2301