1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * NVMe over Fabrics TCP target. 4 * Copyright (c) 2018 Lightbits Labs. All rights reserved. 5 */ 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 #include <linux/module.h> 8 #include <linux/init.h> 9 #include <linux/slab.h> 10 #include <linux/crc32c.h> 11 #include <linux/err.h> 12 #include <linux/nvme-tcp.h> 13 #include <linux/nvme-keyring.h> 14 #include <net/sock.h> 15 #include <net/tcp.h> 16 #include <net/tls.h> 17 #include <net/tls_prot.h> 18 #include <net/handshake.h> 19 #include <linux/inet.h> 20 #include <linux/llist.h> 21 #include <trace/events/sock.h> 22 23 #include "nvmet.h" 24 25 #define NVMET_TCP_DEF_INLINE_DATA_SIZE (4 * PAGE_SIZE) 26 #define NVMET_TCP_MAXH2CDATA 0x400000 /* 16M arbitrary limit */ 27 #define NVMET_TCP_BACKLOG 128 28 29 static int param_store_val(const char *str, int *val, int min, int max) 30 { 31 int ret, new_val; 32 33 ret = kstrtoint(str, 10, &new_val); 34 if (ret) 35 return -EINVAL; 36 37 if (new_val < min || new_val > max) 38 return -EINVAL; 39 40 *val = new_val; 41 return 0; 42 } 43 44 static int set_params(const char *str, const struct kernel_param *kp) 45 { 46 return param_store_val(str, kp->arg, 0, INT_MAX); 47 } 48 49 static const struct kernel_param_ops set_param_ops = { 50 .set = set_params, 51 .get = param_get_int, 52 }; 53 54 /* Define the socket priority to use for connections were it is desirable 55 * that the NIC consider performing optimized packet processing or filtering. 56 * A non-zero value being sufficient to indicate general consideration of any 57 * possible optimization. Making it a module param allows for alternative 58 * values that may be unique for some NIC implementations. 59 */ 60 static int so_priority; 61 device_param_cb(so_priority, &set_param_ops, &so_priority, 0644); 62 MODULE_PARM_DESC(so_priority, "nvmet tcp socket optimize priority: Default 0"); 63 64 /* Define a time period (in usecs) that io_work() shall sample an activated 65 * queue before determining it to be idle. This optional module behavior 66 * can enable NIC solutions that support socket optimized packet processing 67 * using advanced interrupt moderation techniques. 68 */ 69 static int idle_poll_period_usecs; 70 device_param_cb(idle_poll_period_usecs, &set_param_ops, 71 &idle_poll_period_usecs, 0644); 72 MODULE_PARM_DESC(idle_poll_period_usecs, 73 "nvmet tcp io_work poll till idle time period in usecs: Default 0"); 74 75 #ifdef CONFIG_NVME_TARGET_TCP_TLS 76 /* 77 * TLS handshake timeout 78 */ 79 static int tls_handshake_timeout = 10; 80 module_param(tls_handshake_timeout, int, 0644); 81 MODULE_PARM_DESC(tls_handshake_timeout, 82 "nvme TLS handshake timeout in seconds (default 10)"); 83 #endif 84 85 #define NVMET_TCP_RECV_BUDGET 8 86 #define NVMET_TCP_SEND_BUDGET 8 87 #define NVMET_TCP_IO_WORK_BUDGET 64 88 89 enum nvmet_tcp_send_state { 90 NVMET_TCP_SEND_DATA_PDU, 91 NVMET_TCP_SEND_DATA, 92 NVMET_TCP_SEND_R2T, 93 NVMET_TCP_SEND_DDGST, 94 NVMET_TCP_SEND_RESPONSE 95 }; 96 97 enum nvmet_tcp_recv_state { 98 NVMET_TCP_RECV_PDU, 99 NVMET_TCP_RECV_DATA, 100 NVMET_TCP_RECV_DDGST, 101 NVMET_TCP_RECV_ERR, 102 }; 103 104 enum { 105 NVMET_TCP_F_INIT_FAILED = (1 << 0), 106 NVMET_TCP_F_R2T_SENT = (1 << 1), 107 }; 108 109 struct nvmet_tcp_cmd { 110 struct nvmet_tcp_queue *queue; 111 struct nvmet_req req; 112 113 struct nvme_tcp_cmd_pdu *cmd_pdu; 114 struct nvme_tcp_rsp_pdu *rsp_pdu; 115 struct nvme_tcp_data_pdu *data_pdu; 116 struct nvme_tcp_r2t_pdu *r2t_pdu; 117 118 u32 rbytes_done; 119 u32 wbytes_done; 120 121 u32 pdu_len; 122 u32 pdu_recv; 123 int sg_idx; 124 char recv_cbuf[CMSG_LEN(sizeof(char))]; 125 struct msghdr recv_msg; 126 struct bio_vec *iov; 127 u32 flags; 128 129 struct list_head entry; 130 struct llist_node lentry; 131 132 /* send state */ 133 u32 offset; 134 struct scatterlist *cur_sg; 135 enum nvmet_tcp_send_state state; 136 137 __le32 exp_ddgst; 138 __le32 recv_ddgst; 139 }; 140 141 enum nvmet_tcp_queue_state { 142 NVMET_TCP_Q_CONNECTING, 143 NVMET_TCP_Q_TLS_HANDSHAKE, 144 NVMET_TCP_Q_LIVE, 145 NVMET_TCP_Q_DISCONNECTING, 146 NVMET_TCP_Q_FAILED, 147 }; 148 149 struct nvmet_tcp_queue { 150 struct socket *sock; 151 struct nvmet_tcp_port *port; 152 struct work_struct io_work; 153 struct nvmet_cq nvme_cq; 154 struct nvmet_sq nvme_sq; 155 struct kref kref; 156 157 /* send state */ 158 struct nvmet_tcp_cmd *cmds; 159 unsigned int nr_cmds; 160 struct list_head free_list; 161 struct llist_head resp_list; 162 struct list_head resp_send_list; 163 int send_list_len; 164 struct nvmet_tcp_cmd *snd_cmd; 165 166 /* recv state */ 167 int offset; 168 int left; 169 enum nvmet_tcp_recv_state rcv_state; 170 struct nvmet_tcp_cmd *cmd; 171 union nvme_tcp_pdu pdu; 172 173 /* digest state */ 174 bool hdr_digest; 175 bool data_digest; 176 177 /* TLS state */ 178 key_serial_t tls_pskid; 179 struct delayed_work tls_handshake_tmo_work; 180 181 unsigned long poll_end; 182 183 spinlock_t state_lock; 184 enum nvmet_tcp_queue_state state; 185 186 struct sockaddr_storage sockaddr; 187 struct sockaddr_storage sockaddr_peer; 188 struct work_struct release_work; 189 190 int idx; 191 struct list_head queue_list; 192 193 struct nvmet_tcp_cmd connect; 194 195 struct page_frag_cache pf_cache; 196 197 void (*data_ready)(struct sock *); 198 void (*state_change)(struct sock *); 199 void (*write_space)(struct sock *); 200 }; 201 202 struct nvmet_tcp_port { 203 struct socket *sock; 204 struct work_struct accept_work; 205 struct nvmet_port *nport; 206 struct sockaddr_storage addr; 207 void (*data_ready)(struct sock *); 208 }; 209 210 static DEFINE_IDA(nvmet_tcp_queue_ida); 211 static LIST_HEAD(nvmet_tcp_queue_list); 212 static DEFINE_MUTEX(nvmet_tcp_queue_mutex); 213 214 static struct workqueue_struct *nvmet_tcp_wq; 215 static const struct nvmet_fabrics_ops nvmet_tcp_ops; 216 static void nvmet_tcp_free_cmd(struct nvmet_tcp_cmd *c); 217 static void nvmet_tcp_free_cmd_buffers(struct nvmet_tcp_cmd *cmd); 218 219 static inline u16 nvmet_tcp_cmd_tag(struct nvmet_tcp_queue *queue, 220 struct nvmet_tcp_cmd *cmd) 221 { 222 if (unlikely(!queue->nr_cmds)) { 223 /* We didn't allocate cmds yet, send 0xffff */ 224 return USHRT_MAX; 225 } 226 227 return cmd - queue->cmds; 228 } 229 230 static inline bool nvmet_tcp_has_data_in(struct nvmet_tcp_cmd *cmd) 231 { 232 return nvme_is_write(cmd->req.cmd) && 233 cmd->rbytes_done < cmd->req.transfer_len; 234 } 235 236 static inline bool nvmet_tcp_need_data_in(struct nvmet_tcp_cmd *cmd) 237 { 238 return nvmet_tcp_has_data_in(cmd) && !cmd->req.cqe->status; 239 } 240 241 static inline bool nvmet_tcp_need_data_out(struct nvmet_tcp_cmd *cmd) 242 { 243 return !nvme_is_write(cmd->req.cmd) && 244 cmd->req.transfer_len > 0 && 245 !cmd->req.cqe->status; 246 } 247 248 static inline bool nvmet_tcp_has_inline_data(struct nvmet_tcp_cmd *cmd) 249 { 250 return nvme_is_write(cmd->req.cmd) && cmd->pdu_len && 251 !cmd->rbytes_done; 252 } 253 254 static inline struct nvmet_tcp_cmd * 255 nvmet_tcp_get_cmd(struct nvmet_tcp_queue *queue) 256 { 257 struct nvmet_tcp_cmd *cmd; 258 259 cmd = list_first_entry_or_null(&queue->free_list, 260 struct nvmet_tcp_cmd, entry); 261 if (!cmd) 262 return NULL; 263 list_del_init(&cmd->entry); 264 265 cmd->rbytes_done = cmd->wbytes_done = 0; 266 cmd->pdu_len = 0; 267 cmd->pdu_recv = 0; 268 cmd->iov = NULL; 269 cmd->flags = 0; 270 return cmd; 271 } 272 273 static inline void nvmet_tcp_put_cmd(struct nvmet_tcp_cmd *cmd) 274 { 275 if (unlikely(cmd == &cmd->queue->connect)) 276 return; 277 278 list_add_tail(&cmd->entry, &cmd->queue->free_list); 279 } 280 281 static inline int queue_cpu(struct nvmet_tcp_queue *queue) 282 { 283 return queue->sock->sk->sk_incoming_cpu; 284 } 285 286 static inline u8 nvmet_tcp_hdgst_len(struct nvmet_tcp_queue *queue) 287 { 288 return queue->hdr_digest ? NVME_TCP_DIGEST_LENGTH : 0; 289 } 290 291 static inline u8 nvmet_tcp_ddgst_len(struct nvmet_tcp_queue *queue) 292 { 293 return queue->data_digest ? NVME_TCP_DIGEST_LENGTH : 0; 294 } 295 296 static inline void nvmet_tcp_hdgst(void *pdu, size_t len) 297 { 298 put_unaligned_le32(~crc32c(~0, pdu, len), pdu + len); 299 } 300 301 static int nvmet_tcp_verify_hdgst(struct nvmet_tcp_queue *queue, 302 void *pdu, size_t len) 303 { 304 struct nvme_tcp_hdr *hdr = pdu; 305 __le32 recv_digest; 306 __le32 exp_digest; 307 308 if (unlikely(!(hdr->flags & NVME_TCP_F_HDGST))) { 309 pr_err("queue %d: header digest enabled but no header digest\n", 310 queue->idx); 311 return -EPROTO; 312 } 313 314 recv_digest = *(__le32 *)(pdu + hdr->hlen); 315 nvmet_tcp_hdgst(pdu, len); 316 exp_digest = *(__le32 *)(pdu + hdr->hlen); 317 if (recv_digest != exp_digest) { 318 pr_err("queue %d: header digest error: recv %#x expected %#x\n", 319 queue->idx, le32_to_cpu(recv_digest), 320 le32_to_cpu(exp_digest)); 321 return -EPROTO; 322 } 323 324 return 0; 325 } 326 327 static int nvmet_tcp_check_ddgst(struct nvmet_tcp_queue *queue, void *pdu) 328 { 329 struct nvme_tcp_hdr *hdr = pdu; 330 u8 digest_len = nvmet_tcp_hdgst_len(queue); 331 u32 len; 332 333 len = le32_to_cpu(hdr->plen) - hdr->hlen - 334 (hdr->flags & NVME_TCP_F_HDGST ? digest_len : 0); 335 336 if (unlikely(len && !(hdr->flags & NVME_TCP_F_DDGST))) { 337 pr_err("queue %d: data digest flag is cleared\n", queue->idx); 338 return -EPROTO; 339 } 340 341 return 0; 342 } 343 344 /* If cmd buffers are NULL, no operation is performed */ 345 static void nvmet_tcp_free_cmd_buffers(struct nvmet_tcp_cmd *cmd) 346 { 347 kfree(cmd->iov); 348 sgl_free(cmd->req.sg); 349 cmd->iov = NULL; 350 cmd->req.sg = NULL; 351 } 352 353 static int nvmet_tcp_build_pdu_iovec(struct nvmet_tcp_cmd *cmd) 354 { 355 struct bio_vec *iov = cmd->iov; 356 struct scatterlist *sg; 357 u32 length, offset, sg_offset; 358 unsigned int sg_remaining; 359 int nr_pages; 360 361 length = cmd->pdu_len; 362 nr_pages = DIV_ROUND_UP(length, PAGE_SIZE); 363 offset = cmd->rbytes_done; 364 cmd->sg_idx = offset / PAGE_SIZE; 365 sg_offset = offset % PAGE_SIZE; 366 if (!cmd->req.sg_cnt || cmd->sg_idx >= cmd->req.sg_cnt) 367 return -EPROTO; 368 369 sg = &cmd->req.sg[cmd->sg_idx]; 370 sg_remaining = cmd->req.sg_cnt - cmd->sg_idx; 371 372 while (length) { 373 if (!sg_remaining) 374 return -EPROTO; 375 376 if (!sg->length || sg->length <= sg_offset) 377 return -EPROTO; 378 379 u32 iov_len = min_t(u32, length, sg->length - sg_offset); 380 381 bvec_set_page(iov, sg_page(sg), iov_len, 382 sg->offset + sg_offset); 383 384 length -= iov_len; 385 sg = sg_next(sg); 386 sg_remaining--; 387 iov++; 388 sg_offset = 0; 389 } 390 391 iov_iter_bvec(&cmd->recv_msg.msg_iter, ITER_DEST, cmd->iov, 392 nr_pages, cmd->pdu_len); 393 return 0; 394 } 395 396 static void nvmet_tcp_socket_error(struct nvmet_tcp_queue *queue, int status) 397 { 398 /* 399 * Keep rcv_state at RECV_ERR even for the internal -ESHUTDOWN path. 400 * nvmet_tcp_handle_icreq() can return -ESHUTDOWN after the ICReq has 401 * already been consumed and queue teardown has started. 402 * 403 * If nvmet_tcp_data_ready() or nvmet_tcp_write_space() queues 404 * nvmet_tcp_io_work() again before nvmet_tcp_release_queue_work() 405 * cancels it, the queue must not keep that old receive state. 406 * Otherwise the next nvmet_tcp_io_work() run can reach 407 * nvmet_tcp_done_recv_pdu() and try to handle the same ICReq again. 408 * 409 * That is why queue->rcv_state needs to be updated before we return. 410 */ 411 queue->rcv_state = NVMET_TCP_RECV_ERR; 412 if (status == -EPIPE || status == -ECONNRESET || !queue->nvme_sq.ctrl) 413 kernel_sock_shutdown(queue->sock, SHUT_RDWR); 414 else 415 nvmet_ctrl_fatal_error(queue->nvme_sq.ctrl); 416 } 417 418 static int nvmet_tcp_map_data(struct nvmet_tcp_cmd *cmd) 419 { 420 struct nvme_sgl_desc *sgl = &cmd->req.cmd->common.dptr.sgl; 421 u32 len = le32_to_cpu(sgl->length); 422 423 if (!len) 424 return 0; 425 426 /* 427 * inline_data_size only bounds the in-capsule (type 0x01) SGL 428 * descriptor below. A non-inline transport SGL data-block 429 * descriptor skips that check entirely and would otherwise reach 430 * sgl_alloc() with an attacker-controlled len of up to 4 GiB, 431 * pinning that much kernel memory for a command that may never 432 * complete. Bound every descriptor type here, before allocating 433 * anything, using the same ceiling this file already applies to 434 * per-PDU H2C data. 435 */ 436 if (len > NVMET_TCP_MAXH2CDATA) 437 return NVME_SC_SGL_INVALID_DATA | NVME_STATUS_DNR; 438 439 if (sgl->type == ((NVME_SGL_FMT_DATA_DESC << 4) | 440 NVME_SGL_FMT_OFFSET)) { 441 if (!nvme_is_write(cmd->req.cmd)) 442 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 443 444 if (len > cmd->req.port->inline_data_size) 445 return NVME_SC_SGL_INVALID_OFFSET | NVME_STATUS_DNR; 446 cmd->pdu_len = len; 447 } 448 cmd->req.transfer_len += len; 449 450 cmd->req.sg = sgl_alloc(len, GFP_KERNEL | __GFP_NOWARN, 451 &cmd->req.sg_cnt); 452 if (!cmd->req.sg) 453 return NVME_SC_INTERNAL; 454 cmd->cur_sg = cmd->req.sg; 455 456 if (nvmet_tcp_has_data_in(cmd)) { 457 cmd->iov = kmalloc_objs(*cmd->iov, cmd->req.sg_cnt, 458 GFP_KERNEL | __GFP_NOWARN); 459 if (!cmd->iov) 460 goto err; 461 } 462 463 return 0; 464 err: 465 nvmet_tcp_free_cmd_buffers(cmd); 466 return NVME_SC_INTERNAL; 467 } 468 469 static void nvmet_tcp_calc_ddgst(struct nvmet_tcp_cmd *cmd) 470 { 471 size_t total_len = cmd->req.transfer_len; 472 struct scatterlist *sg = cmd->req.sg; 473 u32 crc = ~0; 474 475 while (total_len) { 476 size_t len = min_t(size_t, total_len, sg->length); 477 478 /* 479 * Note that the scatterlist does not contain any highmem pages, 480 * as it was allocated by sgl_alloc() with GFP_KERNEL. 481 */ 482 crc = crc32c(crc, sg_virt(sg), len); 483 total_len -= len; 484 sg = sg_next(sg); 485 } 486 cmd->exp_ddgst = cpu_to_le32(~crc); 487 } 488 489 static void nvmet_setup_c2h_data_pdu(struct nvmet_tcp_cmd *cmd) 490 { 491 struct nvme_tcp_data_pdu *pdu = cmd->data_pdu; 492 struct nvmet_tcp_queue *queue = cmd->queue; 493 u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue); 494 u8 ddgst = nvmet_tcp_ddgst_len(cmd->queue); 495 496 cmd->offset = 0; 497 cmd->state = NVMET_TCP_SEND_DATA_PDU; 498 499 pdu->hdr.type = nvme_tcp_c2h_data; 500 pdu->hdr.flags = NVME_TCP_F_DATA_LAST | (queue->nvme_sq.sqhd_disabled ? 501 NVME_TCP_F_DATA_SUCCESS : 0); 502 pdu->hdr.hlen = sizeof(*pdu); 503 pdu->hdr.pdo = pdu->hdr.hlen + hdgst; 504 pdu->hdr.plen = 505 cpu_to_le32(pdu->hdr.hlen + hdgst + 506 cmd->req.transfer_len + ddgst); 507 pdu->command_id = cmd->req.cqe->command_id; 508 pdu->data_length = cpu_to_le32(cmd->req.transfer_len); 509 pdu->data_offset = cpu_to_le32(cmd->wbytes_done); 510 511 if (queue->data_digest) { 512 pdu->hdr.flags |= NVME_TCP_F_DDGST; 513 nvmet_tcp_calc_ddgst(cmd); 514 } 515 516 if (cmd->queue->hdr_digest) { 517 pdu->hdr.flags |= NVME_TCP_F_HDGST; 518 nvmet_tcp_hdgst(pdu, sizeof(*pdu)); 519 } 520 } 521 522 static void nvmet_setup_r2t_pdu(struct nvmet_tcp_cmd *cmd) 523 { 524 struct nvme_tcp_r2t_pdu *pdu = cmd->r2t_pdu; 525 u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue); 526 527 cmd->offset = 0; 528 cmd->state = NVMET_TCP_SEND_R2T; 529 530 pdu->hdr.type = nvme_tcp_r2t; 531 pdu->hdr.flags = 0; 532 pdu->hdr.hlen = sizeof(*pdu); 533 pdu->hdr.pdo = 0; 534 pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst); 535 536 pdu->command_id = cmd->req.cmd->common.command_id; 537 pdu->ttag = nvmet_tcp_cmd_tag(cmd->queue, cmd); 538 pdu->r2t_length = cpu_to_le32(cmd->req.transfer_len - cmd->rbytes_done); 539 pdu->r2t_offset = cpu_to_le32(cmd->rbytes_done); 540 if (cmd->queue->hdr_digest) { 541 pdu->hdr.flags |= NVME_TCP_F_HDGST; 542 nvmet_tcp_hdgst(pdu, sizeof(*pdu)); 543 } 544 } 545 546 static void nvmet_setup_response_pdu(struct nvmet_tcp_cmd *cmd) 547 { 548 struct nvme_tcp_rsp_pdu *pdu = cmd->rsp_pdu; 549 u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue); 550 551 cmd->offset = 0; 552 cmd->state = NVMET_TCP_SEND_RESPONSE; 553 554 pdu->hdr.type = nvme_tcp_rsp; 555 pdu->hdr.flags = 0; 556 pdu->hdr.hlen = sizeof(*pdu); 557 pdu->hdr.pdo = 0; 558 pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst); 559 if (cmd->queue->hdr_digest) { 560 pdu->hdr.flags |= NVME_TCP_F_HDGST; 561 nvmet_tcp_hdgst(pdu, sizeof(*pdu)); 562 } 563 } 564 565 static void nvmet_tcp_process_resp_list(struct nvmet_tcp_queue *queue) 566 { 567 struct llist_node *node; 568 struct nvmet_tcp_cmd *cmd; 569 570 for (node = llist_del_all(&queue->resp_list); node; node = node->next) { 571 cmd = llist_entry(node, struct nvmet_tcp_cmd, lentry); 572 list_add(&cmd->entry, &queue->resp_send_list); 573 queue->send_list_len++; 574 } 575 } 576 577 static struct nvmet_tcp_cmd *nvmet_tcp_fetch_cmd(struct nvmet_tcp_queue *queue) 578 { 579 queue->snd_cmd = list_first_entry_or_null(&queue->resp_send_list, 580 struct nvmet_tcp_cmd, entry); 581 if (!queue->snd_cmd) { 582 nvmet_tcp_process_resp_list(queue); 583 queue->snd_cmd = 584 list_first_entry_or_null(&queue->resp_send_list, 585 struct nvmet_tcp_cmd, entry); 586 if (unlikely(!queue->snd_cmd)) 587 return NULL; 588 } 589 590 list_del_init(&queue->snd_cmd->entry); 591 queue->send_list_len--; 592 593 if (nvmet_tcp_need_data_out(queue->snd_cmd)) 594 nvmet_setup_c2h_data_pdu(queue->snd_cmd); 595 else if (nvmet_tcp_need_data_in(queue->snd_cmd)) 596 nvmet_setup_r2t_pdu(queue->snd_cmd); 597 else 598 nvmet_setup_response_pdu(queue->snd_cmd); 599 600 return queue->snd_cmd; 601 } 602 603 static void nvmet_tcp_queue_response(struct nvmet_req *req) 604 { 605 struct nvmet_tcp_cmd *cmd = 606 container_of(req, struct nvmet_tcp_cmd, req); 607 struct nvmet_tcp_queue *queue = cmd->queue; 608 enum nvmet_tcp_recv_state queue_state; 609 struct nvmet_tcp_cmd *queue_cmd; 610 struct nvme_sgl_desc *sgl; 611 u32 len; 612 613 /* Pairs with store_release in nvmet_prepare_receive_pdu() */ 614 queue_state = smp_load_acquire(&queue->rcv_state); 615 queue_cmd = READ_ONCE(queue->cmd); 616 617 if (unlikely(cmd == queue_cmd)) { 618 sgl = &cmd->req.cmd->common.dptr.sgl; 619 len = le32_to_cpu(sgl->length); 620 621 /* 622 * Wait for inline data before processing the response. 623 * Avoid using helpers, this might happen before 624 * nvmet_req_init is completed. 625 */ 626 if (queue_state == NVMET_TCP_RECV_PDU && 627 len && len <= cmd->req.port->inline_data_size && 628 nvme_is_write(cmd->req.cmd)) 629 return; 630 } 631 632 llist_add(&cmd->lentry, &queue->resp_list); 633 queue_work_on(queue_cpu(queue), nvmet_tcp_wq, &cmd->queue->io_work); 634 } 635 636 static void nvmet_tcp_execute_request(struct nvmet_tcp_cmd *cmd) 637 { 638 if (unlikely(cmd->flags & NVMET_TCP_F_INIT_FAILED)) 639 nvmet_tcp_queue_response(&cmd->req); 640 else 641 cmd->req.execute(&cmd->req); 642 } 643 644 static int nvmet_try_send_data_pdu(struct nvmet_tcp_cmd *cmd) 645 { 646 struct msghdr msg = { 647 .msg_flags = MSG_DONTWAIT | MSG_MORE | MSG_SPLICE_PAGES, 648 }; 649 struct bio_vec bvec; 650 u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue); 651 int left = sizeof(*cmd->data_pdu) - cmd->offset + hdgst; 652 int ret; 653 654 bvec_set_virt(&bvec, (void *)cmd->data_pdu + cmd->offset, left); 655 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, left); 656 ret = sock_sendmsg(cmd->queue->sock, &msg); 657 if (ret <= 0) 658 return ret; 659 660 cmd->offset += ret; 661 left -= ret; 662 663 if (left) 664 return -EAGAIN; 665 666 cmd->state = NVMET_TCP_SEND_DATA; 667 cmd->offset = 0; 668 return 1; 669 } 670 671 static int nvmet_try_send_data(struct nvmet_tcp_cmd *cmd, bool last_in_batch) 672 { 673 struct nvmet_tcp_queue *queue = cmd->queue; 674 int ret; 675 676 while (cmd->cur_sg) { 677 struct msghdr msg = { 678 .msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES, 679 }; 680 struct page *page = sg_page(cmd->cur_sg); 681 struct bio_vec bvec; 682 u32 left = cmd->cur_sg->length - cmd->offset; 683 684 if ((!last_in_batch && cmd->queue->send_list_len) || 685 cmd->wbytes_done + left < cmd->req.transfer_len || 686 queue->data_digest || !queue->nvme_sq.sqhd_disabled) 687 msg.msg_flags |= MSG_MORE; 688 689 bvec_set_page(&bvec, page, left, cmd->offset); 690 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, left); 691 ret = sock_sendmsg(cmd->queue->sock, &msg); 692 if (ret <= 0) 693 return ret; 694 695 cmd->offset += ret; 696 cmd->wbytes_done += ret; 697 698 /* Done with sg?*/ 699 if (cmd->offset == cmd->cur_sg->length) { 700 cmd->cur_sg = sg_next(cmd->cur_sg); 701 cmd->offset = 0; 702 } 703 } 704 705 if (queue->data_digest) { 706 cmd->state = NVMET_TCP_SEND_DDGST; 707 cmd->offset = 0; 708 } else { 709 if (queue->nvme_sq.sqhd_disabled) { 710 cmd->queue->snd_cmd = NULL; 711 nvmet_tcp_put_cmd(cmd); 712 } else { 713 nvmet_setup_response_pdu(cmd); 714 } 715 } 716 717 if (queue->nvme_sq.sqhd_disabled) 718 nvmet_tcp_free_cmd_buffers(cmd); 719 720 return 1; 721 722 } 723 724 static int nvmet_try_send_response(struct nvmet_tcp_cmd *cmd, 725 bool last_in_batch) 726 { 727 struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES, }; 728 struct bio_vec bvec; 729 u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue); 730 int left = sizeof(*cmd->rsp_pdu) - cmd->offset + hdgst; 731 int ret; 732 733 if (!last_in_batch && cmd->queue->send_list_len) 734 msg.msg_flags |= MSG_MORE; 735 else 736 msg.msg_flags |= MSG_EOR; 737 738 bvec_set_virt(&bvec, (void *)cmd->rsp_pdu + cmd->offset, left); 739 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, left); 740 ret = sock_sendmsg(cmd->queue->sock, &msg); 741 if (ret <= 0) 742 return ret; 743 cmd->offset += ret; 744 left -= ret; 745 746 if (left) 747 return -EAGAIN; 748 749 nvmet_tcp_free_cmd_buffers(cmd); 750 cmd->queue->snd_cmd = NULL; 751 nvmet_tcp_put_cmd(cmd); 752 return 1; 753 } 754 755 static int nvmet_try_send_r2t(struct nvmet_tcp_cmd *cmd, bool last_in_batch) 756 { 757 struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES, }; 758 struct bio_vec bvec; 759 u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue); 760 int left = sizeof(*cmd->r2t_pdu) - cmd->offset + hdgst; 761 int ret; 762 763 if (!last_in_batch && cmd->queue->send_list_len) 764 msg.msg_flags |= MSG_MORE; 765 else 766 msg.msg_flags |= MSG_EOR; 767 768 bvec_set_virt(&bvec, (void *)cmd->r2t_pdu + cmd->offset, left); 769 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, left); 770 ret = sock_sendmsg(cmd->queue->sock, &msg); 771 if (ret <= 0) 772 return ret; 773 cmd->offset += ret; 774 left -= ret; 775 776 if (left) 777 return -EAGAIN; 778 779 cmd->queue->snd_cmd = NULL; 780 cmd->flags |= NVMET_TCP_F_R2T_SENT; 781 return 1; 782 } 783 784 static int nvmet_try_send_ddgst(struct nvmet_tcp_cmd *cmd, bool last_in_batch) 785 { 786 struct nvmet_tcp_queue *queue = cmd->queue; 787 int left = NVME_TCP_DIGEST_LENGTH - cmd->offset; 788 struct msghdr msg = { .msg_flags = MSG_DONTWAIT }; 789 struct kvec iov = { 790 .iov_base = (u8 *)&cmd->exp_ddgst + cmd->offset, 791 .iov_len = left 792 }; 793 int ret; 794 795 if (!last_in_batch && cmd->queue->send_list_len) 796 msg.msg_flags |= MSG_MORE; 797 else 798 msg.msg_flags |= MSG_EOR; 799 800 ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len); 801 if (unlikely(ret <= 0)) 802 return ret; 803 804 cmd->offset += ret; 805 left -= ret; 806 807 if (left) 808 return -EAGAIN; 809 810 if (queue->nvme_sq.sqhd_disabled) { 811 cmd->queue->snd_cmd = NULL; 812 nvmet_tcp_put_cmd(cmd); 813 } else { 814 nvmet_setup_response_pdu(cmd); 815 } 816 return 1; 817 } 818 819 static int nvmet_tcp_try_send_one(struct nvmet_tcp_queue *queue, 820 bool last_in_batch) 821 { 822 struct nvmet_tcp_cmd *cmd = queue->snd_cmd; 823 int ret = 0; 824 825 if (!cmd || queue->state == NVMET_TCP_Q_DISCONNECTING) { 826 cmd = nvmet_tcp_fetch_cmd(queue); 827 if (unlikely(!cmd)) 828 return 0; 829 } 830 831 if (cmd->state == NVMET_TCP_SEND_DATA_PDU) { 832 ret = nvmet_try_send_data_pdu(cmd); 833 if (ret <= 0) 834 goto done_send; 835 } 836 837 if (cmd->state == NVMET_TCP_SEND_DATA) { 838 ret = nvmet_try_send_data(cmd, last_in_batch); 839 if (ret <= 0) 840 goto done_send; 841 } 842 843 if (cmd->state == NVMET_TCP_SEND_DDGST) { 844 ret = nvmet_try_send_ddgst(cmd, last_in_batch); 845 if (ret <= 0) 846 goto done_send; 847 } 848 849 if (cmd->state == NVMET_TCP_SEND_R2T) { 850 ret = nvmet_try_send_r2t(cmd, last_in_batch); 851 if (ret <= 0) 852 goto done_send; 853 } 854 855 if (cmd->state == NVMET_TCP_SEND_RESPONSE) 856 ret = nvmet_try_send_response(cmd, last_in_batch); 857 858 done_send: 859 if (ret < 0) { 860 if (ret == -EAGAIN) 861 return 0; 862 return ret; 863 } 864 865 return 1; 866 } 867 868 static int nvmet_tcp_try_send(struct nvmet_tcp_queue *queue, 869 int budget, int *sends) 870 { 871 int i, ret = 0; 872 873 for (i = 0; i < budget; i++) { 874 ret = nvmet_tcp_try_send_one(queue, i == budget - 1); 875 if (unlikely(ret < 0)) { 876 nvmet_tcp_socket_error(queue, ret); 877 goto done; 878 } else if (ret == 0) { 879 break; 880 } 881 (*sends)++; 882 } 883 done: 884 return ret; 885 } 886 887 static void nvmet_prepare_receive_pdu(struct nvmet_tcp_queue *queue) 888 { 889 queue->offset = 0; 890 queue->left = sizeof(struct nvme_tcp_hdr); 891 WRITE_ONCE(queue->cmd, NULL); 892 /* Ensure rcv_state is visible only after queue->cmd is set */ 893 smp_store_release(&queue->rcv_state, NVMET_TCP_RECV_PDU); 894 } 895 896 static int nvmet_tcp_handle_icreq(struct nvmet_tcp_queue *queue) 897 { 898 struct nvme_tcp_icreq_pdu *icreq = &queue->pdu.icreq; 899 struct nvme_tcp_icresp_pdu *icresp = &queue->pdu.icresp; 900 struct msghdr msg = {}; 901 struct kvec iov; 902 int ret; 903 904 if (le32_to_cpu(icreq->hdr.plen) != sizeof(struct nvme_tcp_icreq_pdu)) { 905 pr_err("bad nvme-tcp pdu length (%d)\n", 906 le32_to_cpu(icreq->hdr.plen)); 907 return -EPROTO; 908 } 909 910 if (icreq->pfv != NVME_TCP_PFV_1_0) { 911 pr_err("queue %d: bad pfv %d\n", queue->idx, icreq->pfv); 912 return -EPROTO; 913 } 914 915 if (icreq->hpda != 0) { 916 pr_err("queue %d: unsupported hpda %d\n", queue->idx, 917 icreq->hpda); 918 return -EPROTO; 919 } 920 921 queue->hdr_digest = !!(icreq->digest & NVME_TCP_HDR_DIGEST_ENABLE); 922 queue->data_digest = !!(icreq->digest & NVME_TCP_DATA_DIGEST_ENABLE); 923 924 memset(icresp, 0, sizeof(*icresp)); 925 icresp->hdr.type = nvme_tcp_icresp; 926 icresp->hdr.hlen = sizeof(*icresp); 927 icresp->hdr.pdo = 0; 928 icresp->hdr.plen = cpu_to_le32(icresp->hdr.hlen); 929 icresp->pfv = cpu_to_le16(NVME_TCP_PFV_1_0); 930 icresp->maxdata = cpu_to_le32(NVMET_TCP_MAXH2CDATA); 931 icresp->cpda = 0; 932 if (queue->hdr_digest) 933 icresp->digest |= NVME_TCP_HDR_DIGEST_ENABLE; 934 if (queue->data_digest) 935 icresp->digest |= NVME_TCP_DATA_DIGEST_ENABLE; 936 937 iov.iov_base = icresp; 938 iov.iov_len = sizeof(*icresp); 939 ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len); 940 if (ret < 0) { 941 spin_lock_bh(&queue->state_lock); 942 if (queue->state == NVMET_TCP_Q_DISCONNECTING) { 943 spin_unlock_bh(&queue->state_lock); 944 return -ESHUTDOWN; 945 } 946 queue->state = NVMET_TCP_Q_FAILED; 947 spin_unlock_bh(&queue->state_lock); 948 return ret; /* queue removal will cleanup */ 949 } 950 951 spin_lock_bh(&queue->state_lock); 952 if (queue->state == NVMET_TCP_Q_DISCONNECTING) { 953 spin_unlock_bh(&queue->state_lock); 954 /* Tell nvmet_tcp_socket_error() teardown is in progress. */ 955 return -ESHUTDOWN; 956 } 957 queue->state = NVMET_TCP_Q_LIVE; 958 spin_unlock_bh(&queue->state_lock); 959 nvmet_prepare_receive_pdu(queue); 960 return 0; 961 } 962 963 static int nvmet_tcp_handle_req_failure(struct nvmet_tcp_queue *queue, 964 struct nvmet_tcp_cmd *cmd, struct nvmet_req *req) 965 { 966 size_t data_len = le32_to_cpu(req->cmd->common.dptr.sgl.length); 967 int ret; 968 969 /* 970 * This command has not been processed yet, hence we are trying to 971 * figure out if there is still pending data left to receive. If 972 * we don't, we can simply prepare for the next pdu and bail out, 973 * otherwise we will need to prepare a buffer and receive the 974 * stale data before continuing forward. 975 */ 976 if (!nvme_is_write(cmd->req.cmd) || !data_len || 977 data_len > cmd->req.port->inline_data_size) { 978 nvmet_prepare_receive_pdu(queue); 979 return 0; 980 } 981 982 ret = nvmet_tcp_map_data(cmd); 983 if (unlikely(ret)) { 984 pr_err("queue %d: failed to map data\n", queue->idx); 985 return -EPROTO; 986 } 987 988 queue->rcv_state = NVMET_TCP_RECV_DATA; 989 cmd->flags |= NVMET_TCP_F_INIT_FAILED; 990 ret = nvmet_tcp_build_pdu_iovec(cmd); 991 if (unlikely(ret)) 992 pr_err("queue %d: failed to build PDU iovec\n", queue->idx); 993 994 return ret; 995 } 996 997 static int nvmet_tcp_handle_h2c_data_pdu(struct nvmet_tcp_queue *queue) 998 { 999 struct nvme_tcp_data_pdu *data = &queue->pdu.data; 1000 struct nvmet_tcp_cmd *cmd; 1001 unsigned int exp_data_len; 1002 1003 if (likely(queue->nr_cmds)) { 1004 if (unlikely(data->ttag >= queue->nr_cmds)) { 1005 pr_err("queue %d: received out of bound ttag %u, nr_cmds %u\n", 1006 queue->idx, data->ttag, queue->nr_cmds); 1007 goto err_proto; 1008 } 1009 cmd = &queue->cmds[data->ttag]; 1010 } else { 1011 cmd = &queue->connect; 1012 } 1013 1014 if (unlikely(!(cmd->flags & NVMET_TCP_F_R2T_SENT))) { 1015 pr_err("queue %d: unsolicited H2CData (ttag %u)\n", 1016 queue->idx, data->ttag); 1017 goto err_proto; 1018 } 1019 1020 if (le32_to_cpu(data->data_offset) != cmd->rbytes_done) { 1021 pr_err("ttag %u unexpected data offset %u (expected %u)\n", 1022 data->ttag, le32_to_cpu(data->data_offset), 1023 cmd->rbytes_done); 1024 goto err_proto; 1025 } 1026 1027 exp_data_len = le32_to_cpu(data->hdr.plen) - 1028 nvmet_tcp_hdgst_len(queue) - 1029 nvmet_tcp_ddgst_len(queue) - 1030 sizeof(*data); 1031 1032 cmd->pdu_len = le32_to_cpu(data->data_length); 1033 if (unlikely(cmd->pdu_len != exp_data_len || 1034 cmd->pdu_len == 0 || 1035 cmd->pdu_len > NVMET_TCP_MAXH2CDATA)) { 1036 pr_err("H2CData PDU len %u is invalid\n", cmd->pdu_len); 1037 goto err_proto; 1038 } 1039 /* 1040 * Ensure command data structures are initialized. We must check both 1041 * cmd->req.sg and cmd->iov because they can have different NULL states: 1042 * - Uninitialized commands: both NULL 1043 * - READ commands: cmd->req.sg allocated, cmd->iov NULL 1044 * - WRITE commands: both allocated 1045 */ 1046 if (unlikely(!cmd->req.sg || !cmd->iov)) { 1047 pr_err("queue %d: H2CData PDU received for invalid command state (ttag %u)\n", 1048 queue->idx, data->ttag); 1049 goto err_proto; 1050 } 1051 cmd->pdu_recv = 0; 1052 if (unlikely(nvmet_tcp_build_pdu_iovec(cmd))) { 1053 pr_err("queue %d: failed to build PDU iovec\n", queue->idx); 1054 goto err_proto; 1055 } 1056 queue->cmd = cmd; 1057 queue->rcv_state = NVMET_TCP_RECV_DATA; 1058 1059 return 0; 1060 1061 err_proto: 1062 /* FIXME: use proper transport errors */ 1063 return -EPROTO; 1064 } 1065 1066 static int nvmet_tcp_done_recv_pdu(struct nvmet_tcp_queue *queue) 1067 { 1068 struct nvme_tcp_hdr *hdr = &queue->pdu.cmd.hdr; 1069 struct nvme_command *nvme_cmd = &queue->pdu.cmd.cmd; 1070 struct nvmet_req *req; 1071 int ret; 1072 1073 if (unlikely(queue->state == NVMET_TCP_Q_CONNECTING)) { 1074 if (hdr->type != nvme_tcp_icreq) { 1075 pr_err("unexpected pdu type (%d) before icreq\n", 1076 hdr->type); 1077 return -EPROTO; 1078 } 1079 return nvmet_tcp_handle_icreq(queue); 1080 } 1081 1082 if (unlikely(hdr->type == nvme_tcp_icreq)) { 1083 pr_err("queue %d: received icreq pdu in state %d\n", 1084 queue->idx, queue->state); 1085 return -EPROTO; 1086 } 1087 1088 if (hdr->type == nvme_tcp_h2c_data) { 1089 ret = nvmet_tcp_handle_h2c_data_pdu(queue); 1090 if (unlikely(ret)) 1091 return ret; 1092 return 0; 1093 } 1094 1095 queue->cmd = nvmet_tcp_get_cmd(queue); 1096 if (unlikely(!queue->cmd)) { 1097 /* This should never happen */ 1098 pr_err("queue %d: out of commands (%d) send_list_len: %d, opcode: %d", 1099 queue->idx, queue->nr_cmds, queue->send_list_len, 1100 nvme_cmd->common.opcode); 1101 return -ENOMEM; 1102 } 1103 1104 req = &queue->cmd->req; 1105 memcpy(req->cmd, nvme_cmd, sizeof(*nvme_cmd)); 1106 1107 if (unlikely(!nvmet_req_init(req, &queue->nvme_sq, &nvmet_tcp_ops))) { 1108 pr_err("failed cmd %p id %d opcode %d, data_len: %d, status: %04x\n", 1109 req->cmd, req->cmd->common.command_id, 1110 req->cmd->common.opcode, 1111 le32_to_cpu(req->cmd->common.dptr.sgl.length), 1112 le16_to_cpu(req->cqe->status)); 1113 1114 return nvmet_tcp_handle_req_failure(queue, queue->cmd, req); 1115 } 1116 1117 ret = nvmet_tcp_map_data(queue->cmd); 1118 if (unlikely(ret)) { 1119 pr_err("queue %d: failed to map data\n", queue->idx); 1120 if (nvmet_tcp_has_inline_data(queue->cmd)) 1121 return -EPROTO; 1122 1123 nvmet_req_complete(req, ret); 1124 ret = -EAGAIN; 1125 goto out; 1126 } 1127 1128 if (nvmet_tcp_need_data_in(queue->cmd)) { 1129 if (nvmet_tcp_has_inline_data(queue->cmd)) { 1130 queue->rcv_state = NVMET_TCP_RECV_DATA; 1131 ret = nvmet_tcp_build_pdu_iovec(queue->cmd); 1132 if (unlikely(ret)) 1133 pr_err("queue %d: failed to build PDU iovec\n", 1134 queue->idx); 1135 return ret; 1136 } 1137 /* send back R2T */ 1138 nvmet_tcp_queue_response(&queue->cmd->req); 1139 goto out; 1140 } 1141 1142 queue->cmd->req.execute(&queue->cmd->req); 1143 out: 1144 nvmet_prepare_receive_pdu(queue); 1145 return ret; 1146 } 1147 1148 static const u8 nvme_tcp_pdu_sizes[] = { 1149 [nvme_tcp_icreq] = sizeof(struct nvme_tcp_icreq_pdu), 1150 [nvme_tcp_cmd] = sizeof(struct nvme_tcp_cmd_pdu), 1151 [nvme_tcp_h2c_data] = sizeof(struct nvme_tcp_data_pdu), 1152 }; 1153 1154 static inline u8 nvmet_tcp_pdu_size(u8 type) 1155 { 1156 size_t idx = type; 1157 1158 return (idx < ARRAY_SIZE(nvme_tcp_pdu_sizes) && 1159 nvme_tcp_pdu_sizes[idx]) ? 1160 nvme_tcp_pdu_sizes[idx] : 0; 1161 } 1162 1163 static inline bool nvmet_tcp_pdu_valid(u8 type) 1164 { 1165 switch (type) { 1166 case nvme_tcp_icreq: 1167 case nvme_tcp_cmd: 1168 case nvme_tcp_h2c_data: 1169 /* fallthru */ 1170 return true; 1171 } 1172 1173 return false; 1174 } 1175 1176 static int nvmet_tcp_tls_record_ok(struct nvmet_tcp_queue *queue, 1177 struct msghdr *msg, char *cbuf) 1178 { 1179 struct cmsghdr *cmsg = (struct cmsghdr *)cbuf; 1180 u8 ctype, level, description; 1181 int ret = 0; 1182 1183 ctype = tls_get_record_type(queue->sock->sk, cmsg); 1184 switch (ctype) { 1185 case 0: 1186 break; 1187 case TLS_RECORD_TYPE_DATA: 1188 break; 1189 case TLS_RECORD_TYPE_ALERT: 1190 tls_alert_recv(queue->sock->sk, msg, &level, &description); 1191 if (level == TLS_ALERT_LEVEL_FATAL) { 1192 pr_err("queue %d: TLS Alert desc %u\n", 1193 queue->idx, description); 1194 ret = -ENOTCONN; 1195 } else { 1196 pr_warn("queue %d: TLS Alert desc %u\n", 1197 queue->idx, description); 1198 ret = -EAGAIN; 1199 } 1200 break; 1201 default: 1202 /* discard this record type */ 1203 pr_err("queue %d: TLS record %d unhandled\n", 1204 queue->idx, ctype); 1205 ret = -EAGAIN; 1206 break; 1207 } 1208 return ret; 1209 } 1210 1211 static int nvmet_tcp_try_recv_pdu(struct nvmet_tcp_queue *queue) 1212 { 1213 struct nvme_tcp_hdr *hdr = &queue->pdu.cmd.hdr; 1214 int len, ret; 1215 struct kvec iov; 1216 char cbuf[CMSG_LEN(sizeof(char))] = {}; 1217 struct msghdr msg = { .msg_flags = MSG_DONTWAIT }; 1218 1219 recv: 1220 iov.iov_base = (void *)&queue->pdu + queue->offset; 1221 iov.iov_len = queue->left; 1222 if (queue->tls_pskid) { 1223 msg.msg_control = cbuf; 1224 msg.msg_controllen = sizeof(cbuf); 1225 } 1226 len = kernel_recvmsg(queue->sock, &msg, &iov, 1, 1227 iov.iov_len, msg.msg_flags); 1228 if (unlikely(len < 0)) 1229 return len; 1230 if (queue->tls_pskid) { 1231 ret = nvmet_tcp_tls_record_ok(queue, &msg, cbuf); 1232 if (ret < 0) 1233 return ret; 1234 } 1235 1236 queue->offset += len; 1237 queue->left -= len; 1238 if (queue->left) 1239 return -EAGAIN; 1240 1241 if (queue->offset == sizeof(struct nvme_tcp_hdr)) { 1242 u8 hdgst = nvmet_tcp_hdgst_len(queue); 1243 1244 if (unlikely(!nvmet_tcp_pdu_valid(hdr->type))) { 1245 pr_err("unexpected pdu type %d\n", hdr->type); 1246 return -EIO; 1247 } 1248 1249 if (unlikely(hdr->hlen != nvmet_tcp_pdu_size(hdr->type))) { 1250 pr_err("pdu %d bad hlen %d\n", hdr->type, hdr->hlen); 1251 return -EIO; 1252 } 1253 1254 queue->left = hdr->hlen - queue->offset + hdgst; 1255 if (queue->left > sizeof(queue->pdu) - queue->offset) 1256 return -EPROTO; 1257 goto recv; 1258 } 1259 1260 if (queue->hdr_digest && 1261 nvmet_tcp_verify_hdgst(queue, &queue->pdu, hdr->hlen)) 1262 return -EPROTO; 1263 1264 if (queue->data_digest && 1265 nvmet_tcp_check_ddgst(queue, &queue->pdu)) 1266 return -EPROTO; 1267 1268 return nvmet_tcp_done_recv_pdu(queue); 1269 } 1270 1271 static void nvmet_tcp_prep_recv_ddgst(struct nvmet_tcp_cmd *cmd) 1272 { 1273 struct nvmet_tcp_queue *queue = cmd->queue; 1274 1275 nvmet_tcp_calc_ddgst(cmd); 1276 queue->offset = 0; 1277 queue->left = NVME_TCP_DIGEST_LENGTH; 1278 queue->rcv_state = NVMET_TCP_RECV_DDGST; 1279 } 1280 1281 static int nvmet_tcp_try_recv_data(struct nvmet_tcp_queue *queue) 1282 { 1283 struct nvmet_tcp_cmd *cmd = queue->cmd; 1284 int len, ret; 1285 1286 while (msg_data_left(&cmd->recv_msg)) { 1287 len = sock_recvmsg(cmd->queue->sock, &cmd->recv_msg, 1288 cmd->recv_msg.msg_flags); 1289 if (len <= 0) 1290 return len; 1291 if (queue->tls_pskid) { 1292 ret = nvmet_tcp_tls_record_ok(cmd->queue, 1293 &cmd->recv_msg, cmd->recv_cbuf); 1294 if (ret < 0) 1295 return ret; 1296 } 1297 1298 cmd->pdu_recv += len; 1299 cmd->rbytes_done += len; 1300 } 1301 1302 if (queue->data_digest) { 1303 nvmet_tcp_prep_recv_ddgst(cmd); 1304 return 0; 1305 } 1306 1307 if (cmd->rbytes_done == cmd->req.transfer_len) 1308 nvmet_tcp_execute_request(cmd); 1309 1310 nvmet_prepare_receive_pdu(queue); 1311 return 0; 1312 } 1313 1314 static int nvmet_tcp_try_recv_ddgst(struct nvmet_tcp_queue *queue) 1315 { 1316 struct nvmet_tcp_cmd *cmd = queue->cmd; 1317 int ret, len; 1318 char cbuf[CMSG_LEN(sizeof(char))] = {}; 1319 struct msghdr msg = { .msg_flags = MSG_DONTWAIT }; 1320 struct kvec iov = { 1321 .iov_base = (void *)&cmd->recv_ddgst + queue->offset, 1322 .iov_len = queue->left 1323 }; 1324 1325 if (queue->tls_pskid) { 1326 msg.msg_control = cbuf; 1327 msg.msg_controllen = sizeof(cbuf); 1328 } 1329 len = kernel_recvmsg(queue->sock, &msg, &iov, 1, 1330 iov.iov_len, msg.msg_flags); 1331 if (unlikely(len < 0)) 1332 return len; 1333 if (queue->tls_pskid) { 1334 ret = nvmet_tcp_tls_record_ok(queue, &msg, cbuf); 1335 if (ret < 0) 1336 return ret; 1337 } 1338 1339 queue->offset += len; 1340 queue->left -= len; 1341 if (queue->left) 1342 return -EAGAIN; 1343 1344 if (queue->data_digest && cmd->exp_ddgst != cmd->recv_ddgst) { 1345 pr_err("queue %d: cmd %d pdu (%d) data digest error: recv %#x expected %#x\n", 1346 queue->idx, cmd->req.cmd->common.command_id, 1347 queue->pdu.cmd.hdr.type, le32_to_cpu(cmd->recv_ddgst), 1348 le32_to_cpu(cmd->exp_ddgst)); 1349 if (!(cmd->flags & NVMET_TCP_F_INIT_FAILED)) { 1350 cmd->req.cqe->status = NVME_SC_CMD_SEQ_ERROR; 1351 nvmet_req_uninit(&cmd->req); 1352 } 1353 nvmet_tcp_free_cmd_buffers(cmd); 1354 ret = -EPROTO; 1355 goto out; 1356 } 1357 1358 if (cmd->rbytes_done == cmd->req.transfer_len) 1359 nvmet_tcp_execute_request(cmd); 1360 1361 ret = 0; 1362 out: 1363 nvmet_prepare_receive_pdu(queue); 1364 return ret; 1365 } 1366 1367 static int nvmet_tcp_try_recv_one(struct nvmet_tcp_queue *queue) 1368 { 1369 int result = 0; 1370 1371 if (unlikely(queue->rcv_state == NVMET_TCP_RECV_ERR)) 1372 return 0; 1373 1374 if (queue->rcv_state == NVMET_TCP_RECV_PDU) { 1375 result = nvmet_tcp_try_recv_pdu(queue); 1376 if (result != 0) 1377 goto done_recv; 1378 } 1379 1380 if (queue->rcv_state == NVMET_TCP_RECV_DATA) { 1381 result = nvmet_tcp_try_recv_data(queue); 1382 if (result != 0) 1383 goto done_recv; 1384 } 1385 1386 if (queue->rcv_state == NVMET_TCP_RECV_DDGST) { 1387 result = nvmet_tcp_try_recv_ddgst(queue); 1388 if (result != 0) 1389 goto done_recv; 1390 } 1391 1392 done_recv: 1393 if (result < 0) { 1394 if (result == -EAGAIN) 1395 return 0; 1396 return result; 1397 } 1398 return 1; 1399 } 1400 1401 static int nvmet_tcp_try_recv(struct nvmet_tcp_queue *queue, 1402 int budget, int *recvs) 1403 { 1404 int i, ret = 0; 1405 1406 for (i = 0; i < budget; i++) { 1407 ret = nvmet_tcp_try_recv_one(queue); 1408 if (unlikely(ret < 0)) { 1409 nvmet_tcp_socket_error(queue, ret); 1410 goto done; 1411 } else if (ret == 0) { 1412 break; 1413 } 1414 (*recvs)++; 1415 } 1416 done: 1417 return ret; 1418 } 1419 1420 static void nvmet_tcp_release_queue(struct kref *kref) 1421 { 1422 struct nvmet_tcp_queue *queue = 1423 container_of(kref, struct nvmet_tcp_queue, kref); 1424 1425 WARN_ON(queue->state != NVMET_TCP_Q_DISCONNECTING); 1426 queue_work(nvmet_wq, &queue->release_work); 1427 } 1428 1429 static void nvmet_tcp_schedule_release_queue(struct nvmet_tcp_queue *queue) 1430 { 1431 spin_lock_bh(&queue->state_lock); 1432 if (queue->state == NVMET_TCP_Q_TLS_HANDSHAKE) { 1433 /* Socket closed during handshake */ 1434 tls_handshake_cancel(queue->sock->sk); 1435 } 1436 if (queue->state != NVMET_TCP_Q_DISCONNECTING) { 1437 queue->state = NVMET_TCP_Q_DISCONNECTING; 1438 kref_put(&queue->kref, nvmet_tcp_release_queue); 1439 } 1440 spin_unlock_bh(&queue->state_lock); 1441 } 1442 1443 static inline void nvmet_tcp_arm_queue_deadline(struct nvmet_tcp_queue *queue) 1444 { 1445 queue->poll_end = jiffies + usecs_to_jiffies(idle_poll_period_usecs); 1446 } 1447 1448 static bool nvmet_tcp_check_queue_deadline(struct nvmet_tcp_queue *queue, 1449 int ops) 1450 { 1451 if (!idle_poll_period_usecs) 1452 return false; 1453 1454 if (ops) 1455 nvmet_tcp_arm_queue_deadline(queue); 1456 1457 return !time_after(jiffies, queue->poll_end); 1458 } 1459 1460 static void nvmet_tcp_io_work(struct work_struct *w) 1461 { 1462 struct nvmet_tcp_queue *queue = 1463 container_of(w, struct nvmet_tcp_queue, io_work); 1464 bool pending; 1465 int ret, ops = 0; 1466 1467 do { 1468 pending = false; 1469 1470 ret = nvmet_tcp_try_recv(queue, NVMET_TCP_RECV_BUDGET, &ops); 1471 if (ret > 0) 1472 pending = true; 1473 else if (ret < 0) 1474 return; 1475 1476 ret = nvmet_tcp_try_send(queue, NVMET_TCP_SEND_BUDGET, &ops); 1477 if (ret > 0) 1478 pending = true; 1479 else if (ret < 0) 1480 return; 1481 1482 } while (pending && ops < NVMET_TCP_IO_WORK_BUDGET); 1483 1484 /* 1485 * Requeue the worker if idle deadline period is in progress or any 1486 * ops activity was recorded during the do-while loop above. 1487 */ 1488 if (nvmet_tcp_check_queue_deadline(queue, ops) || pending) 1489 queue_work_on(queue_cpu(queue), nvmet_tcp_wq, &queue->io_work); 1490 } 1491 1492 static int nvmet_tcp_alloc_cmd(struct nvmet_tcp_queue *queue, 1493 struct nvmet_tcp_cmd *c) 1494 { 1495 u8 hdgst = nvmet_tcp_hdgst_len(queue); 1496 1497 c->queue = queue; 1498 c->req.port = queue->port->nport; 1499 1500 c->cmd_pdu = page_frag_alloc(&queue->pf_cache, 1501 sizeof(*c->cmd_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO); 1502 if (!c->cmd_pdu) 1503 return -ENOMEM; 1504 c->req.cmd = &c->cmd_pdu->cmd; 1505 1506 c->rsp_pdu = page_frag_alloc(&queue->pf_cache, 1507 sizeof(*c->rsp_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO); 1508 if (!c->rsp_pdu) 1509 goto out_free_cmd; 1510 c->req.cqe = &c->rsp_pdu->cqe; 1511 1512 c->data_pdu = page_frag_alloc(&queue->pf_cache, 1513 sizeof(*c->data_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO); 1514 if (!c->data_pdu) 1515 goto out_free_rsp; 1516 1517 c->r2t_pdu = page_frag_alloc(&queue->pf_cache, 1518 sizeof(*c->r2t_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO); 1519 if (!c->r2t_pdu) 1520 goto out_free_data; 1521 1522 if (queue->state == NVMET_TCP_Q_TLS_HANDSHAKE) { 1523 c->recv_msg.msg_control = c->recv_cbuf; 1524 c->recv_msg.msg_controllen = sizeof(c->recv_cbuf); 1525 } 1526 c->recv_msg.msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL; 1527 1528 list_add_tail(&c->entry, &queue->free_list); 1529 1530 return 0; 1531 out_free_data: 1532 page_frag_free(c->data_pdu); 1533 out_free_rsp: 1534 page_frag_free(c->rsp_pdu); 1535 out_free_cmd: 1536 page_frag_free(c->cmd_pdu); 1537 return -ENOMEM; 1538 } 1539 1540 static void nvmet_tcp_free_cmd(struct nvmet_tcp_cmd *c) 1541 { 1542 page_frag_free(c->r2t_pdu); 1543 page_frag_free(c->data_pdu); 1544 page_frag_free(c->rsp_pdu); 1545 page_frag_free(c->cmd_pdu); 1546 } 1547 1548 static int nvmet_tcp_alloc_cmds(struct nvmet_tcp_queue *queue) 1549 { 1550 struct nvmet_tcp_cmd *cmds; 1551 int i, ret = -EINVAL, nr_cmds = queue->nr_cmds; 1552 1553 cmds = kvzalloc_objs(struct nvmet_tcp_cmd, nr_cmds); 1554 if (!cmds) 1555 goto out; 1556 1557 for (i = 0; i < nr_cmds; i++) { 1558 ret = nvmet_tcp_alloc_cmd(queue, cmds + i); 1559 if (ret) 1560 goto out_free; 1561 } 1562 1563 queue->cmds = cmds; 1564 1565 return 0; 1566 out_free: 1567 while (--i >= 0) 1568 nvmet_tcp_free_cmd(cmds + i); 1569 kvfree(cmds); 1570 out: 1571 return ret; 1572 } 1573 1574 static void nvmet_tcp_free_cmds(struct nvmet_tcp_queue *queue) 1575 { 1576 struct nvmet_tcp_cmd *cmds = queue->cmds; 1577 int i; 1578 1579 for (i = 0; i < queue->nr_cmds; i++) 1580 nvmet_tcp_free_cmd(cmds + i); 1581 1582 nvmet_tcp_free_cmd(&queue->connect); 1583 kvfree(cmds); 1584 } 1585 1586 static void nvmet_tcp_restore_socket_callbacks(struct nvmet_tcp_queue *queue) 1587 { 1588 struct socket *sock = queue->sock; 1589 1590 if (!queue->state_change) 1591 return; 1592 1593 write_lock_bh(&sock->sk->sk_callback_lock); 1594 sock->sk->sk_data_ready = queue->data_ready; 1595 sock->sk->sk_state_change = queue->state_change; 1596 sock->sk->sk_write_space = queue->write_space; 1597 sock->sk->sk_user_data = NULL; 1598 write_unlock_bh(&sock->sk->sk_callback_lock); 1599 } 1600 1601 static void nvmet_tcp_uninit_data_in_cmds(struct nvmet_tcp_queue *queue) 1602 { 1603 struct nvmet_tcp_cmd *cmd = queue->cmds; 1604 int i; 1605 1606 for (i = 0; i < queue->nr_cmds; i++, cmd++) { 1607 if (nvmet_tcp_need_data_in(cmd)) 1608 nvmet_req_uninit(&cmd->req); 1609 } 1610 1611 if (!queue->nr_cmds && nvmet_tcp_need_data_in(&queue->connect)) { 1612 /* failed in connect */ 1613 nvmet_req_uninit(&queue->connect.req); 1614 } 1615 } 1616 1617 static void nvmet_tcp_free_cmd_data_in_buffers(struct nvmet_tcp_queue *queue) 1618 { 1619 struct nvmet_tcp_cmd *cmd = queue->cmds; 1620 int i; 1621 1622 for (i = 0; i < queue->nr_cmds; i++, cmd++) 1623 nvmet_tcp_free_cmd_buffers(cmd); 1624 nvmet_tcp_free_cmd_buffers(&queue->connect); 1625 } 1626 1627 static void nvmet_tcp_release_queue_work(struct work_struct *w) 1628 { 1629 struct nvmet_tcp_queue *queue = 1630 container_of(w, struct nvmet_tcp_queue, release_work); 1631 1632 mutex_lock(&nvmet_tcp_queue_mutex); 1633 list_del_init(&queue->queue_list); 1634 mutex_unlock(&nvmet_tcp_queue_mutex); 1635 1636 nvmet_tcp_restore_socket_callbacks(queue); 1637 cancel_delayed_work_sync(&queue->tls_handshake_tmo_work); 1638 cancel_work_sync(&queue->io_work); 1639 /* stop accepting incoming data */ 1640 queue->rcv_state = NVMET_TCP_RECV_ERR; 1641 1642 nvmet_sq_put_tls_key(&queue->nvme_sq); 1643 nvmet_tcp_uninit_data_in_cmds(queue); 1644 nvmet_sq_destroy(&queue->nvme_sq); 1645 nvmet_cq_put(&queue->nvme_cq); 1646 cancel_work_sync(&queue->io_work); 1647 nvmet_tcp_free_cmd_data_in_buffers(queue); 1648 /* ->sock will be released by fput() */ 1649 fput(queue->sock->file); 1650 nvmet_tcp_free_cmds(queue); 1651 ida_free(&nvmet_tcp_queue_ida, queue->idx); 1652 page_frag_cache_drain(&queue->pf_cache); 1653 kfree(queue); 1654 } 1655 1656 static void nvmet_tcp_data_ready(struct sock *sk) 1657 { 1658 struct nvmet_tcp_queue *queue; 1659 1660 trace_sk_data_ready(sk); 1661 1662 read_lock_bh(&sk->sk_callback_lock); 1663 queue = sk->sk_user_data; 1664 if (likely(queue)) { 1665 if (queue->data_ready) 1666 queue->data_ready(sk); 1667 if (queue->state != NVMET_TCP_Q_TLS_HANDSHAKE) 1668 queue_work_on(queue_cpu(queue), nvmet_tcp_wq, 1669 &queue->io_work); 1670 } 1671 read_unlock_bh(&sk->sk_callback_lock); 1672 } 1673 1674 static void nvmet_tcp_write_space(struct sock *sk) 1675 { 1676 struct nvmet_tcp_queue *queue; 1677 1678 read_lock_bh(&sk->sk_callback_lock); 1679 queue = sk->sk_user_data; 1680 if (unlikely(!queue)) 1681 goto out; 1682 1683 if (unlikely(queue->state == NVMET_TCP_Q_CONNECTING)) { 1684 queue->write_space(sk); 1685 goto out; 1686 } 1687 1688 if (sk_stream_is_writeable(sk)) { 1689 clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 1690 queue_work_on(queue_cpu(queue), nvmet_tcp_wq, &queue->io_work); 1691 } 1692 out: 1693 read_unlock_bh(&sk->sk_callback_lock); 1694 } 1695 1696 static void nvmet_tcp_state_change(struct sock *sk) 1697 { 1698 struct nvmet_tcp_queue *queue; 1699 1700 read_lock_bh(&sk->sk_callback_lock); 1701 queue = sk->sk_user_data; 1702 if (!queue) 1703 goto done; 1704 1705 switch (sk->sk_state) { 1706 case TCP_FIN_WAIT2: 1707 case TCP_LAST_ACK: 1708 case TCP_CLOSING: 1709 break; 1710 case TCP_FIN_WAIT1: 1711 case TCP_CLOSE_WAIT: 1712 case TCP_CLOSE: 1713 /* FALLTHRU */ 1714 nvmet_tcp_schedule_release_queue(queue); 1715 break; 1716 default: 1717 pr_warn("queue %d unhandled state %d\n", 1718 queue->idx, sk->sk_state); 1719 } 1720 done: 1721 read_unlock_bh(&sk->sk_callback_lock); 1722 } 1723 1724 static int nvmet_tcp_set_queue_sock(struct nvmet_tcp_queue *queue) 1725 { 1726 struct socket *sock = queue->sock; 1727 struct inet_sock *inet = inet_sk(sock->sk); 1728 int ret; 1729 1730 ret = kernel_getsockname(sock, 1731 (struct sockaddr *)&queue->sockaddr); 1732 if (ret < 0) 1733 return ret; 1734 1735 ret = kernel_getpeername(sock, 1736 (struct sockaddr *)&queue->sockaddr_peer); 1737 if (ret < 0) 1738 return ret; 1739 1740 /* 1741 * Cleanup whatever is sitting in the TCP transmit queue on socket 1742 * close. This is done to prevent stale data from being sent should 1743 * the network connection be restored before TCP times out. 1744 */ 1745 sock_no_linger(sock->sk); 1746 1747 if (so_priority > 0) 1748 sock_set_priority(sock->sk, so_priority); 1749 1750 /* Set socket type of service */ 1751 if (inet->rcv_tos > 0) 1752 ip_sock_set_tos(sock->sk, inet->rcv_tos); 1753 1754 ret = 0; 1755 write_lock_bh(&sock->sk->sk_callback_lock); 1756 if (sock->sk->sk_state != TCP_ESTABLISHED) { 1757 /* 1758 * If the socket is already closing, don't even start 1759 * consuming it 1760 */ 1761 ret = -ENOTCONN; 1762 } else { 1763 sock->sk->sk_user_data = queue; 1764 queue->data_ready = sock->sk->sk_data_ready; 1765 sock->sk->sk_data_ready = nvmet_tcp_data_ready; 1766 queue->state_change = sock->sk->sk_state_change; 1767 sock->sk->sk_state_change = nvmet_tcp_state_change; 1768 queue->write_space = sock->sk->sk_write_space; 1769 sock->sk->sk_write_space = nvmet_tcp_write_space; 1770 if (idle_poll_period_usecs) 1771 nvmet_tcp_arm_queue_deadline(queue); 1772 queue_work_on(queue_cpu(queue), nvmet_tcp_wq, &queue->io_work); 1773 } 1774 write_unlock_bh(&sock->sk->sk_callback_lock); 1775 1776 return ret; 1777 } 1778 1779 #ifdef CONFIG_NVME_TARGET_TCP_TLS 1780 static int nvmet_tcp_try_peek_pdu(struct nvmet_tcp_queue *queue) 1781 { 1782 struct nvme_tcp_hdr *hdr = &queue->pdu.cmd.hdr; 1783 int len, ret; 1784 struct kvec iov = { 1785 .iov_base = (u8 *)&queue->pdu + queue->offset, 1786 .iov_len = sizeof(struct nvme_tcp_hdr), 1787 }; 1788 char cbuf[CMSG_LEN(sizeof(char))] = {}; 1789 struct msghdr msg = { 1790 .msg_control = cbuf, 1791 .msg_controllen = sizeof(cbuf), 1792 .msg_flags = MSG_PEEK, 1793 }; 1794 1795 if (nvmet_port_secure_channel_required(queue->port->nport)) 1796 return 0; 1797 1798 len = kernel_recvmsg(queue->sock, &msg, &iov, 1, 1799 iov.iov_len, msg.msg_flags); 1800 if (unlikely(len < 0)) { 1801 pr_debug("queue %d: peek error %d\n", 1802 queue->idx, len); 1803 return len; 1804 } 1805 1806 ret = nvmet_tcp_tls_record_ok(queue, &msg, cbuf); 1807 if (ret < 0) 1808 return ret; 1809 1810 if (len < sizeof(struct nvme_tcp_hdr)) { 1811 pr_debug("queue %d: short read, %d bytes missing\n", 1812 queue->idx, (int)iov.iov_len - len); 1813 return -EAGAIN; 1814 } 1815 pr_debug("queue %d: hdr type %d hlen %d plen %d size %d\n", 1816 queue->idx, hdr->type, hdr->hlen, hdr->plen, 1817 (int)sizeof(struct nvme_tcp_icreq_pdu)); 1818 if (hdr->type == nvme_tcp_icreq && 1819 hdr->hlen == sizeof(struct nvme_tcp_icreq_pdu) && 1820 hdr->plen == cpu_to_le32(sizeof(struct nvme_tcp_icreq_pdu))) { 1821 pr_debug("queue %d: icreq detected\n", 1822 queue->idx); 1823 return len; 1824 } 1825 return 0; 1826 } 1827 1828 static int nvmet_tcp_tls_key_lookup(struct nvmet_tcp_queue *queue, 1829 key_serial_t peerid) 1830 { 1831 struct key *tls_key = nvme_tls_key_lookup(peerid); 1832 int status = 0; 1833 1834 if (IS_ERR(tls_key)) { 1835 pr_warn("%s: queue %d failed to lookup key %x\n", 1836 __func__, queue->idx, peerid); 1837 spin_lock_bh(&queue->state_lock); 1838 queue->state = NVMET_TCP_Q_FAILED; 1839 spin_unlock_bh(&queue->state_lock); 1840 status = PTR_ERR(tls_key); 1841 } else { 1842 pr_debug("%s: queue %d using TLS PSK %x\n", 1843 __func__, queue->idx, peerid); 1844 queue->nvme_sq.tls_key = tls_key; 1845 } 1846 return status; 1847 } 1848 1849 static void nvmet_tcp_tls_handshake_done(void *data, int status, 1850 key_serial_t peerid) 1851 { 1852 struct nvmet_tcp_queue *queue = data; 1853 1854 pr_debug("queue %d: TLS handshake done, key %x, status %d\n", 1855 queue->idx, peerid, status); 1856 spin_lock_bh(&queue->state_lock); 1857 if (WARN_ON(queue->state != NVMET_TCP_Q_TLS_HANDSHAKE)) { 1858 spin_unlock_bh(&queue->state_lock); 1859 return; 1860 } 1861 if (!status) { 1862 queue->tls_pskid = peerid; 1863 queue->state = NVMET_TCP_Q_CONNECTING; 1864 } else 1865 queue->state = NVMET_TCP_Q_FAILED; 1866 spin_unlock_bh(&queue->state_lock); 1867 1868 cancel_delayed_work_sync(&queue->tls_handshake_tmo_work); 1869 1870 if (!status) 1871 status = nvmet_tcp_tls_key_lookup(queue, peerid); 1872 1873 if (!status) 1874 status = nvmet_tcp_set_queue_sock(queue); 1875 1876 if (status) 1877 nvmet_tcp_schedule_release_queue(queue); 1878 kref_put(&queue->kref, nvmet_tcp_release_queue); 1879 } 1880 1881 static void nvmet_tcp_tls_handshake_timeout(struct work_struct *w) 1882 { 1883 struct nvmet_tcp_queue *queue = container_of(to_delayed_work(w), 1884 struct nvmet_tcp_queue, tls_handshake_tmo_work); 1885 1886 pr_warn("queue %d: TLS handshake timeout\n", queue->idx); 1887 /* 1888 * If tls_handshake_cancel() fails we've lost the race with 1889 * nvmet_tcp_tls_handshake_done() */ 1890 if (!tls_handshake_cancel(queue->sock->sk)) 1891 return; 1892 spin_lock_bh(&queue->state_lock); 1893 if (WARN_ON(queue->state != NVMET_TCP_Q_TLS_HANDSHAKE)) { 1894 spin_unlock_bh(&queue->state_lock); 1895 return; 1896 } 1897 queue->state = NVMET_TCP_Q_FAILED; 1898 spin_unlock_bh(&queue->state_lock); 1899 nvmet_tcp_schedule_release_queue(queue); 1900 kref_put(&queue->kref, nvmet_tcp_release_queue); 1901 } 1902 1903 static int nvmet_tcp_tls_handshake(struct nvmet_tcp_queue *queue) 1904 { 1905 int ret = -EOPNOTSUPP; 1906 struct tls_handshake_args args; 1907 1908 if (queue->state != NVMET_TCP_Q_TLS_HANDSHAKE) { 1909 pr_warn("cannot start TLS in state %d\n", queue->state); 1910 return -EINVAL; 1911 } 1912 1913 kref_get(&queue->kref); 1914 pr_debug("queue %d: TLS ServerHello\n", queue->idx); 1915 memset(&args, 0, sizeof(args)); 1916 args.ta_sock = queue->sock; 1917 args.ta_done = nvmet_tcp_tls_handshake_done; 1918 args.ta_data = queue; 1919 args.ta_keyring = key_serial(queue->port->nport->keyring); 1920 args.ta_timeout_ms = tls_handshake_timeout * 1000; 1921 1922 ret = tls_server_hello_psk(&args, GFP_KERNEL); 1923 if (ret) { 1924 kref_put(&queue->kref, nvmet_tcp_release_queue); 1925 pr_err("failed to start TLS, err=%d\n", ret); 1926 } else { 1927 queue_delayed_work(nvmet_wq, &queue->tls_handshake_tmo_work, 1928 tls_handshake_timeout * HZ); 1929 } 1930 return ret; 1931 } 1932 #else 1933 static void nvmet_tcp_tls_handshake_timeout(struct work_struct *w) {} 1934 #endif 1935 1936 static void nvmet_tcp_alloc_queue(struct nvmet_tcp_port *port, 1937 struct socket *newsock) 1938 { 1939 struct nvmet_tcp_queue *queue; 1940 struct file *sock_file = NULL; 1941 int ret; 1942 1943 queue = kzalloc_obj(*queue); 1944 if (!queue) { 1945 ret = -ENOMEM; 1946 goto out_release; 1947 } 1948 1949 INIT_WORK(&queue->release_work, nvmet_tcp_release_queue_work); 1950 INIT_WORK(&queue->io_work, nvmet_tcp_io_work); 1951 kref_init(&queue->kref); 1952 queue->sock = newsock; 1953 queue->port = port; 1954 queue->nr_cmds = 0; 1955 spin_lock_init(&queue->state_lock); 1956 if (queue->port->nport->disc_addr.tsas.tcp.sectype == 1957 NVMF_TCP_SECTYPE_TLS13) 1958 queue->state = NVMET_TCP_Q_TLS_HANDSHAKE; 1959 else 1960 queue->state = NVMET_TCP_Q_CONNECTING; 1961 INIT_LIST_HEAD(&queue->free_list); 1962 init_llist_head(&queue->resp_list); 1963 INIT_LIST_HEAD(&queue->resp_send_list); 1964 1965 sock_file = sock_alloc_file(queue->sock, O_CLOEXEC, NULL); 1966 if (IS_ERR(sock_file)) { 1967 ret = PTR_ERR(sock_file); 1968 goto out_free_queue; 1969 } 1970 1971 queue->idx = ida_alloc(&nvmet_tcp_queue_ida, GFP_KERNEL); 1972 if (queue->idx < 0) { 1973 ret = queue->idx; 1974 goto out_sock; 1975 } 1976 1977 ret = nvmet_tcp_alloc_cmd(queue, &queue->connect); 1978 if (ret) 1979 goto out_ida_remove; 1980 1981 nvmet_cq_init(&queue->nvme_cq); 1982 ret = nvmet_sq_init(&queue->nvme_sq, &queue->nvme_cq); 1983 if (ret) 1984 goto out_free_connect; 1985 1986 nvmet_prepare_receive_pdu(queue); 1987 1988 mutex_lock(&nvmet_tcp_queue_mutex); 1989 list_add_tail(&queue->queue_list, &nvmet_tcp_queue_list); 1990 mutex_unlock(&nvmet_tcp_queue_mutex); 1991 1992 INIT_DELAYED_WORK(&queue->tls_handshake_tmo_work, 1993 nvmet_tcp_tls_handshake_timeout); 1994 #ifdef CONFIG_NVME_TARGET_TCP_TLS 1995 if (queue->state == NVMET_TCP_Q_TLS_HANDSHAKE) { 1996 struct sock *sk = queue->sock->sk; 1997 1998 /* Restore the default callbacks before starting upcall */ 1999 write_lock_bh(&sk->sk_callback_lock); 2000 sk->sk_user_data = NULL; 2001 sk->sk_data_ready = port->data_ready; 2002 write_unlock_bh(&sk->sk_callback_lock); 2003 if (!nvmet_tcp_try_peek_pdu(queue)) { 2004 if (!nvmet_tcp_tls_handshake(queue)) 2005 return; 2006 /* TLS handshake failed, terminate the connection */ 2007 goto out_destroy_sq; 2008 } 2009 /* Not a TLS connection, continue with normal processing */ 2010 queue->state = NVMET_TCP_Q_CONNECTING; 2011 } 2012 #endif 2013 2014 ret = nvmet_tcp_set_queue_sock(queue); 2015 if (ret) 2016 goto out_destroy_sq; 2017 2018 return; 2019 out_destroy_sq: 2020 mutex_lock(&nvmet_tcp_queue_mutex); 2021 list_del_init(&queue->queue_list); 2022 mutex_unlock(&nvmet_tcp_queue_mutex); 2023 nvmet_sq_destroy(&queue->nvme_sq); 2024 out_free_connect: 2025 nvmet_cq_put(&queue->nvme_cq); 2026 nvmet_tcp_free_cmd(&queue->connect); 2027 out_ida_remove: 2028 ida_free(&nvmet_tcp_queue_ida, queue->idx); 2029 /* 2030 * Drain the page fragment cache if any allocations were done. 2031 * The first allocation using pf_cache is nvmet_tcp_alloc_cmd() 2032 * for queue->connect after ida_alloc(). 2033 */ 2034 page_frag_cache_drain(&queue->pf_cache); 2035 out_sock: 2036 fput(queue->sock->file); 2037 out_free_queue: 2038 kfree(queue); 2039 out_release: 2040 pr_err("failed to allocate queue, error %d\n", ret); 2041 if (!sock_file) 2042 sock_release(newsock); 2043 } 2044 2045 static void nvmet_tcp_accept_work(struct work_struct *w) 2046 { 2047 struct nvmet_tcp_port *port = 2048 container_of(w, struct nvmet_tcp_port, accept_work); 2049 struct socket *newsock; 2050 int ret; 2051 2052 while (true) { 2053 ret = kernel_accept(port->sock, &newsock, O_NONBLOCK); 2054 if (ret < 0) { 2055 if (ret != -EAGAIN) 2056 pr_warn("failed to accept err=%d\n", ret); 2057 return; 2058 } 2059 nvmet_tcp_alloc_queue(port, newsock); 2060 } 2061 } 2062 2063 static void nvmet_tcp_listen_data_ready(struct sock *sk) 2064 { 2065 struct nvmet_tcp_port *port; 2066 2067 trace_sk_data_ready(sk); 2068 2069 if (sk->sk_state != TCP_LISTEN) 2070 return; 2071 2072 read_lock_bh(&sk->sk_callback_lock); 2073 port = sk->sk_user_data; 2074 if (port) 2075 queue_work(nvmet_wq, &port->accept_work); 2076 read_unlock_bh(&sk->sk_callback_lock); 2077 } 2078 2079 static int nvmet_tcp_add_port(struct nvmet_port *nport) 2080 { 2081 struct nvmet_tcp_port *port; 2082 __kernel_sa_family_t af; 2083 int ret; 2084 2085 port = kzalloc_obj(*port); 2086 if (!port) 2087 return -ENOMEM; 2088 2089 switch (nport->disc_addr.adrfam) { 2090 case NVMF_ADDR_FAMILY_IP4: 2091 af = AF_INET; 2092 break; 2093 case NVMF_ADDR_FAMILY_IP6: 2094 af = AF_INET6; 2095 break; 2096 default: 2097 pr_err("address family %d not supported\n", 2098 nport->disc_addr.adrfam); 2099 ret = -EINVAL; 2100 goto err_port; 2101 } 2102 2103 ret = inet_pton_with_scope(&init_net, af, nport->disc_addr.traddr, 2104 nport->disc_addr.trsvcid, &port->addr); 2105 if (ret) { 2106 pr_err("malformed ip/port passed: %s:%s\n", 2107 nport->disc_addr.traddr, nport->disc_addr.trsvcid); 2108 goto err_port; 2109 } 2110 2111 port->nport = nport; 2112 INIT_WORK(&port->accept_work, nvmet_tcp_accept_work); 2113 if (port->nport->inline_data_size < 0) 2114 port->nport->inline_data_size = NVMET_TCP_DEF_INLINE_DATA_SIZE; 2115 2116 ret = sock_create(port->addr.ss_family, SOCK_STREAM, 2117 IPPROTO_TCP, &port->sock); 2118 if (ret) { 2119 pr_err("failed to create a socket\n"); 2120 goto err_port; 2121 } 2122 2123 port->sock->sk->sk_user_data = port; 2124 port->data_ready = port->sock->sk->sk_data_ready; 2125 port->sock->sk->sk_data_ready = nvmet_tcp_listen_data_ready; 2126 sock_set_reuseaddr(port->sock->sk); 2127 tcp_sock_set_nodelay(port->sock->sk); 2128 if (so_priority > 0) 2129 sock_set_priority(port->sock->sk, so_priority); 2130 2131 ret = kernel_bind(port->sock, (struct sockaddr_unsized *)&port->addr, 2132 sizeof(port->addr)); 2133 if (ret) { 2134 pr_err("failed to bind port socket %d\n", ret); 2135 goto err_sock; 2136 } 2137 2138 ret = kernel_listen(port->sock, NVMET_TCP_BACKLOG); 2139 if (ret) { 2140 pr_err("failed to listen %d on port sock\n", ret); 2141 goto err_sock; 2142 } 2143 2144 nport->priv = port; 2145 pr_info("enabling port %d (%pISpc)\n", 2146 le16_to_cpu(nport->disc_addr.portid), &port->addr); 2147 2148 return 0; 2149 2150 err_sock: 2151 sock_release(port->sock); 2152 err_port: 2153 kfree(port); 2154 return ret; 2155 } 2156 2157 static void nvmet_tcp_destroy_port_queues(struct nvmet_tcp_port *port) 2158 { 2159 struct nvmet_tcp_queue *queue; 2160 2161 mutex_lock(&nvmet_tcp_queue_mutex); 2162 list_for_each_entry(queue, &nvmet_tcp_queue_list, queue_list) 2163 if (queue->port == port) 2164 kernel_sock_shutdown(queue->sock, SHUT_RDWR); 2165 mutex_unlock(&nvmet_tcp_queue_mutex); 2166 } 2167 2168 static void nvmet_tcp_remove_port(struct nvmet_port *nport) 2169 { 2170 struct nvmet_tcp_port *port = nport->priv; 2171 2172 write_lock_bh(&port->sock->sk->sk_callback_lock); 2173 port->sock->sk->sk_data_ready = port->data_ready; 2174 port->sock->sk->sk_user_data = NULL; 2175 write_unlock_bh(&port->sock->sk->sk_callback_lock); 2176 cancel_work_sync(&port->accept_work); 2177 /* 2178 * Destroy the remaining queues, which are not belong to any 2179 * controller yet. 2180 */ 2181 nvmet_tcp_destroy_port_queues(port); 2182 2183 sock_release(port->sock); 2184 kfree(port); 2185 } 2186 2187 static void nvmet_tcp_delete_ctrl(struct nvmet_ctrl *ctrl) 2188 { 2189 struct nvmet_tcp_queue *queue; 2190 2191 mutex_lock(&nvmet_tcp_queue_mutex); 2192 list_for_each_entry(queue, &nvmet_tcp_queue_list, queue_list) 2193 if (queue->nvme_sq.ctrl == ctrl) 2194 kernel_sock_shutdown(queue->sock, SHUT_RDWR); 2195 mutex_unlock(&nvmet_tcp_queue_mutex); 2196 } 2197 2198 static u16 nvmet_tcp_install_queue(struct nvmet_sq *sq) 2199 { 2200 struct nvmet_tcp_queue *queue = 2201 container_of(sq, struct nvmet_tcp_queue, nvme_sq); 2202 2203 if (sq->qid == 0) { 2204 struct nvmet_tcp_queue *q; 2205 int pending = 0; 2206 2207 /* Check for pending controller teardown */ 2208 mutex_lock(&nvmet_tcp_queue_mutex); 2209 list_for_each_entry(q, &nvmet_tcp_queue_list, queue_list) { 2210 if (q->nvme_sq.ctrl == sq->ctrl && 2211 q->state == NVMET_TCP_Q_DISCONNECTING) 2212 pending++; 2213 } 2214 mutex_unlock(&nvmet_tcp_queue_mutex); 2215 if (pending > NVMET_TCP_BACKLOG) 2216 return NVME_SC_CONNECT_CTRL_BUSY; 2217 } 2218 2219 queue->nr_cmds = sq->size * 2; 2220 if (nvmet_tcp_alloc_cmds(queue)) { 2221 queue->nr_cmds = 0; 2222 return NVME_SC_INTERNAL; 2223 } 2224 return 0; 2225 } 2226 2227 static void nvmet_tcp_disc_port_addr(struct nvmet_req *req, 2228 struct nvmet_port *nport, char *traddr) 2229 { 2230 struct nvmet_tcp_port *port = nport->priv; 2231 2232 if (inet_addr_is_any(&port->addr)) { 2233 struct nvmet_tcp_cmd *cmd = 2234 container_of(req, struct nvmet_tcp_cmd, req); 2235 struct nvmet_tcp_queue *queue = cmd->queue; 2236 2237 sprintf(traddr, "%pISc", (struct sockaddr *)&queue->sockaddr); 2238 } else { 2239 memcpy(traddr, nport->disc_addr.traddr, NVMF_TRADDR_SIZE); 2240 } 2241 } 2242 2243 static ssize_t nvmet_tcp_host_port_addr(struct nvmet_ctrl *ctrl, 2244 char *traddr, size_t traddr_len) 2245 { 2246 struct nvmet_sq *sq = ctrl->sqs[0]; 2247 struct nvmet_tcp_queue *queue = 2248 container_of(sq, struct nvmet_tcp_queue, nvme_sq); 2249 2250 if (queue->sockaddr_peer.ss_family == AF_UNSPEC) 2251 return -EINVAL; 2252 return snprintf(traddr, traddr_len, "%pISc", 2253 (struct sockaddr *)&queue->sockaddr_peer); 2254 } 2255 2256 static const struct nvmet_fabrics_ops nvmet_tcp_ops = { 2257 .owner = THIS_MODULE, 2258 .type = NVMF_TRTYPE_TCP, 2259 .msdbd = 1, 2260 .add_port = nvmet_tcp_add_port, 2261 .remove_port = nvmet_tcp_remove_port, 2262 .queue_response = nvmet_tcp_queue_response, 2263 .delete_ctrl = nvmet_tcp_delete_ctrl, 2264 .install_queue = nvmet_tcp_install_queue, 2265 .disc_traddr = nvmet_tcp_disc_port_addr, 2266 .host_traddr = nvmet_tcp_host_port_addr, 2267 }; 2268 2269 static int __init nvmet_tcp_init(void) 2270 { 2271 int ret; 2272 2273 nvmet_tcp_wq = alloc_workqueue("nvmet_tcp_wq", 2274 WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_PERCPU, 0); 2275 if (!nvmet_tcp_wq) 2276 return -ENOMEM; 2277 2278 ret = nvmet_register_transport(&nvmet_tcp_ops); 2279 if (ret) 2280 goto err; 2281 2282 return 0; 2283 err: 2284 destroy_workqueue(nvmet_tcp_wq); 2285 return ret; 2286 } 2287 2288 static void __exit nvmet_tcp_exit(void) 2289 { 2290 struct nvmet_tcp_queue *queue; 2291 2292 nvmet_unregister_transport(&nvmet_tcp_ops); 2293 2294 flush_workqueue(nvmet_wq); 2295 mutex_lock(&nvmet_tcp_queue_mutex); 2296 list_for_each_entry(queue, &nvmet_tcp_queue_list, queue_list) 2297 kernel_sock_shutdown(queue->sock, SHUT_RDWR); 2298 mutex_unlock(&nvmet_tcp_queue_mutex); 2299 flush_workqueue(nvmet_wq); 2300 2301 destroy_workqueue(nvmet_tcp_wq); 2302 ida_destroy(&nvmet_tcp_queue_ida); 2303 } 2304 2305 module_init(nvmet_tcp_init); 2306 module_exit(nvmet_tcp_exit); 2307 2308 MODULE_DESCRIPTION("NVMe target TCP transport driver"); 2309 MODULE_LICENSE("GPL v2"); 2310 MODULE_ALIAS("nvmet-transport-3"); /* 3 == NVMF_TRTYPE_TCP */ 2311