1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * NVMe over Fabrics TCP host. 4 * Copyright (c) 2018 Lightbits Labs. All rights reserved. 5 */ 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 #include <linux/module.h> 8 #include <linux/init.h> 9 #include <linux/slab.h> 10 #include <linux/err.h> 11 #include <linux/nvme-tcp.h> 12 #include <linux/nvme-keyring.h> 13 #include <net/sock.h> 14 #include <net/tcp.h> 15 #include <net/tls.h> 16 #include <net/tls_prot.h> 17 #include <net/handshake.h> 18 #include <linux/blk-mq.h> 19 #include <crypto/hash.h> 20 #include <net/busy_poll.h> 21 #include <trace/events/sock.h> 22 23 #include "nvme.h" 24 #include "fabrics.h" 25 26 struct nvme_tcp_queue; 27 28 /* Define the socket priority to use for connections were it is desirable 29 * that the NIC consider performing optimized packet processing or filtering. 30 * A non-zero value being sufficient to indicate general consideration of any 31 * possible optimization. Making it a module param allows for alternative 32 * values that may be unique for some NIC implementations. 33 */ 34 static int so_priority; 35 module_param(so_priority, int, 0644); 36 MODULE_PARM_DESC(so_priority, "nvme tcp socket optimize priority"); 37 38 /* 39 * Use the unbound workqueue for nvme_tcp_wq, then we can set the cpu affinity 40 * from sysfs. 41 */ 42 static bool wq_unbound; 43 module_param(wq_unbound, bool, 0644); 44 MODULE_PARM_DESC(wq_unbound, "Use unbound workqueue for nvme-tcp IO context (default false)"); 45 46 /* 47 * TLS handshake timeout 48 */ 49 static int tls_handshake_timeout = 10; 50 #ifdef CONFIG_NVME_TCP_TLS 51 module_param(tls_handshake_timeout, int, 0644); 52 MODULE_PARM_DESC(tls_handshake_timeout, 53 "nvme TLS handshake timeout in seconds (default 10)"); 54 #endif 55 56 static atomic_t nvme_tcp_cpu_queues[NR_CPUS]; 57 58 #ifdef CONFIG_DEBUG_LOCK_ALLOC 59 /* lockdep can detect a circular dependency of the form 60 * sk_lock -> mmap_lock (page fault) -> fs locks -> sk_lock 61 * because dependencies are tracked for both nvme-tcp and user contexts. Using 62 * a separate class prevents lockdep from conflating nvme-tcp socket use with 63 * user-space socket API use. 64 */ 65 static struct lock_class_key nvme_tcp_sk_key[2]; 66 static struct lock_class_key nvme_tcp_slock_key[2]; 67 68 static void nvme_tcp_reclassify_socket(struct socket *sock) 69 { 70 struct sock *sk = sock->sk; 71 72 if (WARN_ON_ONCE(!sock_allow_reclassification(sk))) 73 return; 74 75 switch (sk->sk_family) { 76 case AF_INET: 77 sock_lock_init_class_and_name(sk, "slock-AF_INET-NVME", 78 &nvme_tcp_slock_key[0], 79 "sk_lock-AF_INET-NVME", 80 &nvme_tcp_sk_key[0]); 81 break; 82 case AF_INET6: 83 sock_lock_init_class_and_name(sk, "slock-AF_INET6-NVME", 84 &nvme_tcp_slock_key[1], 85 "sk_lock-AF_INET6-NVME", 86 &nvme_tcp_sk_key[1]); 87 break; 88 default: 89 WARN_ON_ONCE(1); 90 } 91 } 92 #else 93 static void nvme_tcp_reclassify_socket(struct socket *sock) { } 94 #endif 95 96 enum nvme_tcp_send_state { 97 NVME_TCP_SEND_CMD_PDU = 0, 98 NVME_TCP_SEND_H2C_PDU, 99 NVME_TCP_SEND_DATA, 100 NVME_TCP_SEND_DDGST, 101 }; 102 103 struct nvme_tcp_request { 104 struct nvme_request req; 105 void *pdu; 106 struct nvme_tcp_queue *queue; 107 u32 data_len; 108 u32 pdu_len; 109 u32 pdu_sent; 110 u32 h2cdata_left; 111 u32 h2cdata_offset; 112 u16 ttag; 113 __le16 status; 114 struct list_head entry; 115 struct llist_node lentry; 116 __le32 ddgst; 117 118 struct bio *curr_bio; 119 struct iov_iter iter; 120 121 /* send state */ 122 size_t offset; 123 size_t data_sent; 124 enum nvme_tcp_send_state state; 125 }; 126 127 enum nvme_tcp_queue_flags { 128 NVME_TCP_Q_ALLOCATED = 0, 129 NVME_TCP_Q_LIVE = 1, 130 NVME_TCP_Q_POLLING = 2, 131 NVME_TCP_Q_IO_CPU_SET = 3, 132 }; 133 134 enum nvme_tcp_recv_state { 135 NVME_TCP_RECV_PDU = 0, 136 NVME_TCP_RECV_DATA, 137 NVME_TCP_RECV_DDGST, 138 }; 139 140 struct nvme_tcp_ctrl; 141 struct nvme_tcp_queue { 142 struct socket *sock; 143 struct work_struct io_work; 144 int io_cpu; 145 146 struct mutex queue_lock; 147 struct mutex send_mutex; 148 struct llist_head req_list; 149 struct list_head send_list; 150 151 /* recv state */ 152 void *pdu; 153 int pdu_remaining; 154 int pdu_offset; 155 size_t data_remaining; 156 size_t ddgst_remaining; 157 unsigned int nr_cqe; 158 159 /* send state */ 160 struct nvme_tcp_request *request; 161 162 u32 maxh2cdata; 163 size_t cmnd_capsule_len; 164 struct nvme_tcp_ctrl *ctrl; 165 unsigned long flags; 166 bool rd_enabled; 167 168 bool hdr_digest; 169 bool data_digest; 170 bool tls_enabled; 171 struct ahash_request *rcv_hash; 172 struct ahash_request *snd_hash; 173 __le32 exp_ddgst; 174 __le32 recv_ddgst; 175 struct completion tls_complete; 176 int tls_err; 177 struct page_frag_cache pf_cache; 178 179 void (*state_change)(struct sock *); 180 void (*data_ready)(struct sock *); 181 void (*write_space)(struct sock *); 182 }; 183 184 struct nvme_tcp_ctrl { 185 /* read only in the hot path */ 186 struct nvme_tcp_queue *queues; 187 struct blk_mq_tag_set tag_set; 188 189 /* other member variables */ 190 struct list_head list; 191 struct blk_mq_tag_set admin_tag_set; 192 struct sockaddr_storage addr; 193 struct sockaddr_storage src_addr; 194 struct nvme_ctrl ctrl; 195 196 struct work_struct err_work; 197 struct delayed_work connect_work; 198 struct nvme_tcp_request async_req; 199 u32 io_queues[HCTX_MAX_TYPES]; 200 }; 201 202 static LIST_HEAD(nvme_tcp_ctrl_list); 203 static DEFINE_MUTEX(nvme_tcp_ctrl_mutex); 204 static struct workqueue_struct *nvme_tcp_wq; 205 static const struct blk_mq_ops nvme_tcp_mq_ops; 206 static const struct blk_mq_ops nvme_tcp_admin_mq_ops; 207 static int nvme_tcp_try_send(struct nvme_tcp_queue *queue); 208 209 static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl) 210 { 211 return container_of(ctrl, struct nvme_tcp_ctrl, ctrl); 212 } 213 214 static inline int nvme_tcp_queue_id(struct nvme_tcp_queue *queue) 215 { 216 return queue - queue->ctrl->queues; 217 } 218 219 static inline bool nvme_tcp_recv_pdu_supported(enum nvme_tcp_pdu_type type) 220 { 221 switch (type) { 222 case nvme_tcp_c2h_term: 223 case nvme_tcp_c2h_data: 224 case nvme_tcp_r2t: 225 case nvme_tcp_rsp: 226 return true; 227 default: 228 return false; 229 } 230 } 231 232 /* 233 * Check if the queue is TLS encrypted 234 */ 235 static inline bool nvme_tcp_queue_tls(struct nvme_tcp_queue *queue) 236 { 237 if (!IS_ENABLED(CONFIG_NVME_TCP_TLS)) 238 return 0; 239 240 return queue->tls_enabled; 241 } 242 243 /* 244 * Check if TLS is configured for the controller. 245 */ 246 static inline bool nvme_tcp_tls_configured(struct nvme_ctrl *ctrl) 247 { 248 if (!IS_ENABLED(CONFIG_NVME_TCP_TLS)) 249 return 0; 250 251 return ctrl->opts->tls || ctrl->opts->concat; 252 } 253 254 static inline struct blk_mq_tags *nvme_tcp_tagset(struct nvme_tcp_queue *queue) 255 { 256 u32 queue_idx = nvme_tcp_queue_id(queue); 257 258 if (queue_idx == 0) 259 return queue->ctrl->admin_tag_set.tags[queue_idx]; 260 return queue->ctrl->tag_set.tags[queue_idx - 1]; 261 } 262 263 static inline u8 nvme_tcp_hdgst_len(struct nvme_tcp_queue *queue) 264 { 265 return queue->hdr_digest ? NVME_TCP_DIGEST_LENGTH : 0; 266 } 267 268 static inline u8 nvme_tcp_ddgst_len(struct nvme_tcp_queue *queue) 269 { 270 return queue->data_digest ? NVME_TCP_DIGEST_LENGTH : 0; 271 } 272 273 static inline void *nvme_tcp_req_cmd_pdu(struct nvme_tcp_request *req) 274 { 275 return req->pdu; 276 } 277 278 static inline void *nvme_tcp_req_data_pdu(struct nvme_tcp_request *req) 279 { 280 /* use the pdu space in the back for the data pdu */ 281 return req->pdu + sizeof(struct nvme_tcp_cmd_pdu) - 282 sizeof(struct nvme_tcp_data_pdu); 283 } 284 285 static inline size_t nvme_tcp_inline_data_size(struct nvme_tcp_request *req) 286 { 287 if (nvme_is_fabrics(req->req.cmd)) 288 return NVME_TCP_ADMIN_CCSZ; 289 return req->queue->cmnd_capsule_len - sizeof(struct nvme_command); 290 } 291 292 static inline bool nvme_tcp_async_req(struct nvme_tcp_request *req) 293 { 294 return req == &req->queue->ctrl->async_req; 295 } 296 297 static inline bool nvme_tcp_has_inline_data(struct nvme_tcp_request *req) 298 { 299 struct request *rq; 300 301 if (unlikely(nvme_tcp_async_req(req))) 302 return false; /* async events don't have a request */ 303 304 rq = blk_mq_rq_from_pdu(req); 305 306 return rq_data_dir(rq) == WRITE && req->data_len && 307 req->data_len <= nvme_tcp_inline_data_size(req); 308 } 309 310 static inline struct page *nvme_tcp_req_cur_page(struct nvme_tcp_request *req) 311 { 312 return req->iter.bvec->bv_page; 313 } 314 315 static inline size_t nvme_tcp_req_cur_offset(struct nvme_tcp_request *req) 316 { 317 return req->iter.bvec->bv_offset + req->iter.iov_offset; 318 } 319 320 static inline size_t nvme_tcp_req_cur_length(struct nvme_tcp_request *req) 321 { 322 return min_t(size_t, iov_iter_single_seg_count(&req->iter), 323 req->pdu_len - req->pdu_sent); 324 } 325 326 static inline size_t nvme_tcp_pdu_data_left(struct nvme_tcp_request *req) 327 { 328 return rq_data_dir(blk_mq_rq_from_pdu(req)) == WRITE ? 329 req->pdu_len - req->pdu_sent : 0; 330 } 331 332 static inline size_t nvme_tcp_pdu_last_send(struct nvme_tcp_request *req, 333 int len) 334 { 335 return nvme_tcp_pdu_data_left(req) <= len; 336 } 337 338 static void nvme_tcp_init_iter(struct nvme_tcp_request *req, 339 unsigned int dir) 340 { 341 struct request *rq = blk_mq_rq_from_pdu(req); 342 struct bio_vec *vec; 343 unsigned int size; 344 int nr_bvec; 345 size_t offset; 346 347 if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) { 348 vec = &rq->special_vec; 349 nr_bvec = 1; 350 size = blk_rq_payload_bytes(rq); 351 offset = 0; 352 } else { 353 struct bio *bio = req->curr_bio; 354 struct bvec_iter bi; 355 struct bio_vec bv; 356 357 vec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter); 358 nr_bvec = 0; 359 bio_for_each_bvec(bv, bio, bi) { 360 nr_bvec++; 361 } 362 size = bio->bi_iter.bi_size; 363 offset = bio->bi_iter.bi_bvec_done; 364 } 365 366 iov_iter_bvec(&req->iter, dir, vec, nr_bvec, size); 367 req->iter.iov_offset = offset; 368 } 369 370 static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req, 371 int len) 372 { 373 req->data_sent += len; 374 req->pdu_sent += len; 375 iov_iter_advance(&req->iter, len); 376 if (!iov_iter_count(&req->iter) && 377 req->data_sent < req->data_len) { 378 req->curr_bio = req->curr_bio->bi_next; 379 nvme_tcp_init_iter(req, ITER_SOURCE); 380 } 381 } 382 383 static inline void nvme_tcp_send_all(struct nvme_tcp_queue *queue) 384 { 385 int ret; 386 387 /* drain the send queue as much as we can... */ 388 do { 389 ret = nvme_tcp_try_send(queue); 390 } while (ret > 0); 391 } 392 393 static inline bool nvme_tcp_queue_has_pending(struct nvme_tcp_queue *queue) 394 { 395 return !list_empty(&queue->send_list) || 396 !llist_empty(&queue->req_list); 397 } 398 399 static inline bool nvme_tcp_queue_more(struct nvme_tcp_queue *queue) 400 { 401 return !nvme_tcp_queue_tls(queue) && 402 nvme_tcp_queue_has_pending(queue); 403 } 404 405 static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req, 406 bool sync, bool last) 407 { 408 struct nvme_tcp_queue *queue = req->queue; 409 bool empty; 410 411 empty = llist_add(&req->lentry, &queue->req_list) && 412 list_empty(&queue->send_list) && !queue->request; 413 414 /* 415 * if we're the first on the send_list and we can try to send 416 * directly, otherwise queue io_work. Also, only do that if we 417 * are on the same cpu, so we don't introduce contention. 418 */ 419 if (queue->io_cpu == raw_smp_processor_id() && 420 sync && empty && mutex_trylock(&queue->send_mutex)) { 421 nvme_tcp_send_all(queue); 422 mutex_unlock(&queue->send_mutex); 423 } 424 425 if (last && nvme_tcp_queue_has_pending(queue)) 426 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 427 } 428 429 static void nvme_tcp_process_req_list(struct nvme_tcp_queue *queue) 430 { 431 struct nvme_tcp_request *req; 432 struct llist_node *node; 433 434 for (node = llist_del_all(&queue->req_list); node; node = node->next) { 435 req = llist_entry(node, struct nvme_tcp_request, lentry); 436 list_add(&req->entry, &queue->send_list); 437 } 438 } 439 440 static inline struct nvme_tcp_request * 441 nvme_tcp_fetch_request(struct nvme_tcp_queue *queue) 442 { 443 struct nvme_tcp_request *req; 444 445 req = list_first_entry_or_null(&queue->send_list, 446 struct nvme_tcp_request, entry); 447 if (!req) { 448 nvme_tcp_process_req_list(queue); 449 req = list_first_entry_or_null(&queue->send_list, 450 struct nvme_tcp_request, entry); 451 if (unlikely(!req)) 452 return NULL; 453 } 454 455 list_del(&req->entry); 456 return req; 457 } 458 459 static inline void nvme_tcp_ddgst_final(struct ahash_request *hash, 460 __le32 *dgst) 461 { 462 ahash_request_set_crypt(hash, NULL, (u8 *)dgst, 0); 463 crypto_ahash_final(hash); 464 } 465 466 static inline void nvme_tcp_ddgst_update(struct ahash_request *hash, 467 struct page *page, off_t off, size_t len) 468 { 469 struct scatterlist sg; 470 471 sg_init_table(&sg, 1); 472 sg_set_page(&sg, page, len, off); 473 ahash_request_set_crypt(hash, &sg, NULL, len); 474 crypto_ahash_update(hash); 475 } 476 477 static inline void nvme_tcp_hdgst(struct ahash_request *hash, 478 void *pdu, size_t len) 479 { 480 struct scatterlist sg; 481 482 sg_init_one(&sg, pdu, len); 483 ahash_request_set_crypt(hash, &sg, pdu + len, len); 484 crypto_ahash_digest(hash); 485 } 486 487 static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue, 488 void *pdu, size_t pdu_len) 489 { 490 struct nvme_tcp_hdr *hdr = pdu; 491 __le32 recv_digest; 492 __le32 exp_digest; 493 494 if (unlikely(!(hdr->flags & NVME_TCP_F_HDGST))) { 495 dev_err(queue->ctrl->ctrl.device, 496 "queue %d: header digest flag is cleared\n", 497 nvme_tcp_queue_id(queue)); 498 return -EPROTO; 499 } 500 501 recv_digest = *(__le32 *)(pdu + hdr->hlen); 502 nvme_tcp_hdgst(queue->rcv_hash, pdu, pdu_len); 503 exp_digest = *(__le32 *)(pdu + hdr->hlen); 504 if (recv_digest != exp_digest) { 505 dev_err(queue->ctrl->ctrl.device, 506 "header digest error: recv %#x expected %#x\n", 507 le32_to_cpu(recv_digest), le32_to_cpu(exp_digest)); 508 return -EIO; 509 } 510 511 return 0; 512 } 513 514 static int nvme_tcp_check_ddgst(struct nvme_tcp_queue *queue, void *pdu) 515 { 516 struct nvme_tcp_hdr *hdr = pdu; 517 u8 digest_len = nvme_tcp_hdgst_len(queue); 518 u32 len; 519 520 len = le32_to_cpu(hdr->plen) - hdr->hlen - 521 ((hdr->flags & NVME_TCP_F_HDGST) ? digest_len : 0); 522 523 if (unlikely(len && !(hdr->flags & NVME_TCP_F_DDGST))) { 524 dev_err(queue->ctrl->ctrl.device, 525 "queue %d: data digest flag is cleared\n", 526 nvme_tcp_queue_id(queue)); 527 return -EPROTO; 528 } 529 crypto_ahash_init(queue->rcv_hash); 530 531 return 0; 532 } 533 534 static void nvme_tcp_exit_request(struct blk_mq_tag_set *set, 535 struct request *rq, unsigned int hctx_idx) 536 { 537 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 538 539 page_frag_free(req->pdu); 540 } 541 542 static int nvme_tcp_init_request(struct blk_mq_tag_set *set, 543 struct request *rq, unsigned int hctx_idx, 544 unsigned int numa_node) 545 { 546 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(set->driver_data); 547 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 548 struct nvme_tcp_cmd_pdu *pdu; 549 int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0; 550 struct nvme_tcp_queue *queue = &ctrl->queues[queue_idx]; 551 u8 hdgst = nvme_tcp_hdgst_len(queue); 552 553 req->pdu = page_frag_alloc(&queue->pf_cache, 554 sizeof(struct nvme_tcp_cmd_pdu) + hdgst, 555 GFP_KERNEL | __GFP_ZERO); 556 if (!req->pdu) 557 return -ENOMEM; 558 559 pdu = req->pdu; 560 req->queue = queue; 561 nvme_req(rq)->ctrl = &ctrl->ctrl; 562 nvme_req(rq)->cmd = &pdu->cmd; 563 564 return 0; 565 } 566 567 static int nvme_tcp_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, 568 unsigned int hctx_idx) 569 { 570 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(data); 571 struct nvme_tcp_queue *queue = &ctrl->queues[hctx_idx + 1]; 572 573 hctx->driver_data = queue; 574 return 0; 575 } 576 577 static int nvme_tcp_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data, 578 unsigned int hctx_idx) 579 { 580 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(data); 581 struct nvme_tcp_queue *queue = &ctrl->queues[0]; 582 583 hctx->driver_data = queue; 584 return 0; 585 } 586 587 static enum nvme_tcp_recv_state 588 nvme_tcp_recv_state(struct nvme_tcp_queue *queue) 589 { 590 return (queue->pdu_remaining) ? NVME_TCP_RECV_PDU : 591 (queue->ddgst_remaining) ? NVME_TCP_RECV_DDGST : 592 NVME_TCP_RECV_DATA; 593 } 594 595 static void nvme_tcp_init_recv_ctx(struct nvme_tcp_queue *queue) 596 { 597 queue->pdu_remaining = sizeof(struct nvme_tcp_rsp_pdu) + 598 nvme_tcp_hdgst_len(queue); 599 queue->pdu_offset = 0; 600 queue->data_remaining = -1; 601 queue->ddgst_remaining = 0; 602 } 603 604 static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl) 605 { 606 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) 607 return; 608 609 dev_warn(ctrl->device, "starting error recovery\n"); 610 queue_work(nvme_reset_wq, &to_tcp_ctrl(ctrl)->err_work); 611 } 612 613 static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue, 614 struct nvme_completion *cqe) 615 { 616 struct nvme_tcp_request *req; 617 struct request *rq; 618 619 rq = nvme_find_rq(nvme_tcp_tagset(queue), cqe->command_id); 620 if (!rq) { 621 dev_err(queue->ctrl->ctrl.device, 622 "got bad cqe.command_id %#x on queue %d\n", 623 cqe->command_id, nvme_tcp_queue_id(queue)); 624 nvme_tcp_error_recovery(&queue->ctrl->ctrl); 625 return -EINVAL; 626 } 627 628 req = blk_mq_rq_to_pdu(rq); 629 if (req->status == cpu_to_le16(NVME_SC_SUCCESS)) 630 req->status = cqe->status; 631 632 if (!nvme_try_complete_req(rq, req->status, cqe->result)) 633 nvme_complete_rq(rq); 634 queue->nr_cqe++; 635 636 return 0; 637 } 638 639 static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue, 640 struct nvme_tcp_data_pdu *pdu) 641 { 642 struct request *rq; 643 644 rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id); 645 if (!rq) { 646 dev_err(queue->ctrl->ctrl.device, 647 "got bad c2hdata.command_id %#x on queue %d\n", 648 pdu->command_id, nvme_tcp_queue_id(queue)); 649 return -ENOENT; 650 } 651 652 if (!blk_rq_payload_bytes(rq)) { 653 dev_err(queue->ctrl->ctrl.device, 654 "queue %d tag %#x unexpected data\n", 655 nvme_tcp_queue_id(queue), rq->tag); 656 return -EIO; 657 } 658 659 queue->data_remaining = le32_to_cpu(pdu->data_length); 660 661 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS && 662 unlikely(!(pdu->hdr.flags & NVME_TCP_F_DATA_LAST))) { 663 dev_err(queue->ctrl->ctrl.device, 664 "queue %d tag %#x SUCCESS set but not last PDU\n", 665 nvme_tcp_queue_id(queue), rq->tag); 666 nvme_tcp_error_recovery(&queue->ctrl->ctrl); 667 return -EPROTO; 668 } 669 670 return 0; 671 } 672 673 static int nvme_tcp_handle_comp(struct nvme_tcp_queue *queue, 674 struct nvme_tcp_rsp_pdu *pdu) 675 { 676 struct nvme_completion *cqe = &pdu->cqe; 677 int ret = 0; 678 679 /* 680 * AEN requests are special as they don't time out and can 681 * survive any kind of queue freeze and often don't respond to 682 * aborts. We don't even bother to allocate a struct request 683 * for them but rather special case them here. 684 */ 685 if (unlikely(nvme_is_aen_req(nvme_tcp_queue_id(queue), 686 cqe->command_id))) 687 nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status, 688 &cqe->result); 689 else 690 ret = nvme_tcp_process_nvme_cqe(queue, cqe); 691 692 return ret; 693 } 694 695 static void nvme_tcp_setup_h2c_data_pdu(struct nvme_tcp_request *req) 696 { 697 struct nvme_tcp_data_pdu *data = nvme_tcp_req_data_pdu(req); 698 struct nvme_tcp_queue *queue = req->queue; 699 struct request *rq = blk_mq_rq_from_pdu(req); 700 u32 h2cdata_sent = req->pdu_len; 701 u8 hdgst = nvme_tcp_hdgst_len(queue); 702 u8 ddgst = nvme_tcp_ddgst_len(queue); 703 704 req->state = NVME_TCP_SEND_H2C_PDU; 705 req->offset = 0; 706 req->pdu_len = min(req->h2cdata_left, queue->maxh2cdata); 707 req->pdu_sent = 0; 708 req->h2cdata_left -= req->pdu_len; 709 req->h2cdata_offset += h2cdata_sent; 710 711 memset(data, 0, sizeof(*data)); 712 data->hdr.type = nvme_tcp_h2c_data; 713 if (!req->h2cdata_left) 714 data->hdr.flags = NVME_TCP_F_DATA_LAST; 715 if (queue->hdr_digest) 716 data->hdr.flags |= NVME_TCP_F_HDGST; 717 if (queue->data_digest) 718 data->hdr.flags |= NVME_TCP_F_DDGST; 719 data->hdr.hlen = sizeof(*data); 720 data->hdr.pdo = data->hdr.hlen + hdgst; 721 data->hdr.plen = 722 cpu_to_le32(data->hdr.hlen + hdgst + req->pdu_len + ddgst); 723 data->ttag = req->ttag; 724 data->command_id = nvme_cid(rq); 725 data->data_offset = cpu_to_le32(req->h2cdata_offset); 726 data->data_length = cpu_to_le32(req->pdu_len); 727 } 728 729 static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue, 730 struct nvme_tcp_r2t_pdu *pdu) 731 { 732 struct nvme_tcp_request *req; 733 struct request *rq; 734 u32 r2t_length = le32_to_cpu(pdu->r2t_length); 735 u32 r2t_offset = le32_to_cpu(pdu->r2t_offset); 736 737 rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id); 738 if (!rq) { 739 dev_err(queue->ctrl->ctrl.device, 740 "got bad r2t.command_id %#x on queue %d\n", 741 pdu->command_id, nvme_tcp_queue_id(queue)); 742 return -ENOENT; 743 } 744 req = blk_mq_rq_to_pdu(rq); 745 746 if (unlikely(!r2t_length)) { 747 dev_err(queue->ctrl->ctrl.device, 748 "req %d r2t len is %u, probably a bug...\n", 749 rq->tag, r2t_length); 750 return -EPROTO; 751 } 752 753 if (unlikely(req->data_sent + r2t_length > req->data_len)) { 754 dev_err(queue->ctrl->ctrl.device, 755 "req %d r2t len %u exceeded data len %u (%zu sent)\n", 756 rq->tag, r2t_length, req->data_len, req->data_sent); 757 return -EPROTO; 758 } 759 760 if (unlikely(r2t_offset < req->data_sent)) { 761 dev_err(queue->ctrl->ctrl.device, 762 "req %d unexpected r2t offset %u (expected %zu)\n", 763 rq->tag, r2t_offset, req->data_sent); 764 return -EPROTO; 765 } 766 767 req->pdu_len = 0; 768 req->h2cdata_left = r2t_length; 769 req->h2cdata_offset = r2t_offset; 770 req->ttag = pdu->ttag; 771 772 nvme_tcp_setup_h2c_data_pdu(req); 773 nvme_tcp_queue_request(req, false, true); 774 775 return 0; 776 } 777 778 static void nvme_tcp_handle_c2h_term(struct nvme_tcp_queue *queue, 779 struct nvme_tcp_term_pdu *pdu) 780 { 781 u16 fes; 782 const char *msg; 783 u32 plen = le32_to_cpu(pdu->hdr.plen); 784 785 static const char * const msg_table[] = { 786 [NVME_TCP_FES_INVALID_PDU_HDR] = "Invalid PDU Header Field", 787 [NVME_TCP_FES_PDU_SEQ_ERR] = "PDU Sequence Error", 788 [NVME_TCP_FES_HDR_DIGEST_ERR] = "Header Digest Error", 789 [NVME_TCP_FES_DATA_OUT_OF_RANGE] = "Data Transfer Out Of Range", 790 [NVME_TCP_FES_DATA_LIMIT_EXCEEDED] = "Data Transfer Limit Exceeded", 791 [NVME_TCP_FES_UNSUPPORTED_PARAM] = "Unsupported Parameter", 792 }; 793 794 if (plen < NVME_TCP_MIN_C2HTERM_PLEN || 795 plen > NVME_TCP_MAX_C2HTERM_PLEN) { 796 dev_err(queue->ctrl->ctrl.device, 797 "Received a malformed C2HTermReq PDU (plen = %u)\n", 798 plen); 799 return; 800 } 801 802 fes = le16_to_cpu(pdu->fes); 803 if (fes && fes < ARRAY_SIZE(msg_table)) 804 msg = msg_table[fes]; 805 else 806 msg = "Unknown"; 807 808 dev_err(queue->ctrl->ctrl.device, 809 "Received C2HTermReq (FES = %s)\n", msg); 810 } 811 812 static int nvme_tcp_recv_pdu(struct nvme_tcp_queue *queue, struct sk_buff *skb, 813 unsigned int *offset, size_t *len) 814 { 815 struct nvme_tcp_hdr *hdr; 816 char *pdu = queue->pdu; 817 size_t rcv_len = min_t(size_t, *len, queue->pdu_remaining); 818 int ret; 819 820 ret = skb_copy_bits(skb, *offset, 821 &pdu[queue->pdu_offset], rcv_len); 822 if (unlikely(ret)) 823 return ret; 824 825 queue->pdu_remaining -= rcv_len; 826 queue->pdu_offset += rcv_len; 827 *offset += rcv_len; 828 *len -= rcv_len; 829 if (queue->pdu_remaining) 830 return 0; 831 832 hdr = queue->pdu; 833 if (unlikely(hdr->hlen != sizeof(struct nvme_tcp_rsp_pdu))) { 834 if (!nvme_tcp_recv_pdu_supported(hdr->type)) 835 goto unsupported_pdu; 836 837 dev_err(queue->ctrl->ctrl.device, 838 "pdu type %d has unexpected header length (%d)\n", 839 hdr->type, hdr->hlen); 840 return -EPROTO; 841 } 842 843 if (unlikely(hdr->type == nvme_tcp_c2h_term)) { 844 /* 845 * C2HTermReq never includes Header or Data digests. 846 * Skip the checks. 847 */ 848 nvme_tcp_handle_c2h_term(queue, (void *)queue->pdu); 849 return -EINVAL; 850 } 851 852 if (queue->hdr_digest) { 853 ret = nvme_tcp_verify_hdgst(queue, queue->pdu, hdr->hlen); 854 if (unlikely(ret)) 855 return ret; 856 } 857 858 859 if (queue->data_digest) { 860 ret = nvme_tcp_check_ddgst(queue, queue->pdu); 861 if (unlikely(ret)) 862 return ret; 863 } 864 865 switch (hdr->type) { 866 case nvme_tcp_c2h_data: 867 return nvme_tcp_handle_c2h_data(queue, (void *)queue->pdu); 868 case nvme_tcp_rsp: 869 nvme_tcp_init_recv_ctx(queue); 870 return nvme_tcp_handle_comp(queue, (void *)queue->pdu); 871 case nvme_tcp_r2t: 872 nvme_tcp_init_recv_ctx(queue); 873 return nvme_tcp_handle_r2t(queue, (void *)queue->pdu); 874 default: 875 goto unsupported_pdu; 876 } 877 878 unsupported_pdu: 879 dev_err(queue->ctrl->ctrl.device, 880 "unsupported pdu type (%d)\n", hdr->type); 881 return -EINVAL; 882 } 883 884 static inline void nvme_tcp_end_request(struct request *rq, u16 status) 885 { 886 union nvme_result res = {}; 887 888 if (!nvme_try_complete_req(rq, cpu_to_le16(status << 1), res)) 889 nvme_complete_rq(rq); 890 } 891 892 static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb, 893 unsigned int *offset, size_t *len) 894 { 895 struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu; 896 struct request *rq = 897 nvme_cid_to_rq(nvme_tcp_tagset(queue), pdu->command_id); 898 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 899 900 while (true) { 901 int recv_len, ret; 902 903 recv_len = min_t(size_t, *len, queue->data_remaining); 904 if (!recv_len) 905 break; 906 907 if (!iov_iter_count(&req->iter)) { 908 req->curr_bio = req->curr_bio->bi_next; 909 910 /* 911 * If we don`t have any bios it means that controller 912 * sent more data than we requested, hence error 913 */ 914 if (!req->curr_bio) { 915 dev_err(queue->ctrl->ctrl.device, 916 "queue %d no space in request %#x", 917 nvme_tcp_queue_id(queue), rq->tag); 918 nvme_tcp_init_recv_ctx(queue); 919 return -EIO; 920 } 921 nvme_tcp_init_iter(req, ITER_DEST); 922 } 923 924 /* we can read only from what is left in this bio */ 925 recv_len = min_t(size_t, recv_len, 926 iov_iter_count(&req->iter)); 927 928 if (queue->data_digest) 929 ret = skb_copy_and_hash_datagram_iter(skb, *offset, 930 &req->iter, recv_len, queue->rcv_hash); 931 else 932 ret = skb_copy_datagram_iter(skb, *offset, 933 &req->iter, recv_len); 934 if (ret) { 935 dev_err(queue->ctrl->ctrl.device, 936 "queue %d failed to copy request %#x data", 937 nvme_tcp_queue_id(queue), rq->tag); 938 return ret; 939 } 940 941 *len -= recv_len; 942 *offset += recv_len; 943 queue->data_remaining -= recv_len; 944 } 945 946 if (!queue->data_remaining) { 947 if (queue->data_digest) { 948 nvme_tcp_ddgst_final(queue->rcv_hash, &queue->exp_ddgst); 949 queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH; 950 } else { 951 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) { 952 nvme_tcp_end_request(rq, 953 le16_to_cpu(req->status)); 954 queue->nr_cqe++; 955 } 956 nvme_tcp_init_recv_ctx(queue); 957 } 958 } 959 960 return 0; 961 } 962 963 static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue *queue, 964 struct sk_buff *skb, unsigned int *offset, size_t *len) 965 { 966 struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu; 967 char *ddgst = (char *)&queue->recv_ddgst; 968 size_t recv_len = min_t(size_t, *len, queue->ddgst_remaining); 969 off_t off = NVME_TCP_DIGEST_LENGTH - queue->ddgst_remaining; 970 int ret; 971 972 ret = skb_copy_bits(skb, *offset, &ddgst[off], recv_len); 973 if (unlikely(ret)) 974 return ret; 975 976 queue->ddgst_remaining -= recv_len; 977 *offset += recv_len; 978 *len -= recv_len; 979 if (queue->ddgst_remaining) 980 return 0; 981 982 if (queue->recv_ddgst != queue->exp_ddgst) { 983 struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue), 984 pdu->command_id); 985 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 986 987 req->status = cpu_to_le16(NVME_SC_DATA_XFER_ERROR); 988 989 dev_err(queue->ctrl->ctrl.device, 990 "data digest error: recv %#x expected %#x\n", 991 le32_to_cpu(queue->recv_ddgst), 992 le32_to_cpu(queue->exp_ddgst)); 993 } 994 995 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) { 996 struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue), 997 pdu->command_id); 998 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 999 1000 nvme_tcp_end_request(rq, le16_to_cpu(req->status)); 1001 queue->nr_cqe++; 1002 } 1003 1004 nvme_tcp_init_recv_ctx(queue); 1005 return 0; 1006 } 1007 1008 static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb, 1009 unsigned int offset, size_t len) 1010 { 1011 struct nvme_tcp_queue *queue = desc->arg.data; 1012 size_t consumed = len; 1013 int result; 1014 1015 if (unlikely(!queue->rd_enabled)) 1016 return -EFAULT; 1017 1018 while (len) { 1019 switch (nvme_tcp_recv_state(queue)) { 1020 case NVME_TCP_RECV_PDU: 1021 result = nvme_tcp_recv_pdu(queue, skb, &offset, &len); 1022 break; 1023 case NVME_TCP_RECV_DATA: 1024 result = nvme_tcp_recv_data(queue, skb, &offset, &len); 1025 break; 1026 case NVME_TCP_RECV_DDGST: 1027 result = nvme_tcp_recv_ddgst(queue, skb, &offset, &len); 1028 break; 1029 default: 1030 result = -EFAULT; 1031 } 1032 if (result) { 1033 dev_err(queue->ctrl->ctrl.device, 1034 "receive failed: %d\n", result); 1035 queue->rd_enabled = false; 1036 nvme_tcp_error_recovery(&queue->ctrl->ctrl); 1037 return result; 1038 } 1039 } 1040 1041 return consumed; 1042 } 1043 1044 static void nvme_tcp_data_ready(struct sock *sk) 1045 { 1046 struct nvme_tcp_queue *queue; 1047 1048 trace_sk_data_ready(sk); 1049 1050 read_lock_bh(&sk->sk_callback_lock); 1051 queue = sk->sk_user_data; 1052 if (likely(queue && queue->rd_enabled) && 1053 !test_bit(NVME_TCP_Q_POLLING, &queue->flags)) 1054 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 1055 read_unlock_bh(&sk->sk_callback_lock); 1056 } 1057 1058 static void nvme_tcp_write_space(struct sock *sk) 1059 { 1060 struct nvme_tcp_queue *queue; 1061 1062 read_lock_bh(&sk->sk_callback_lock); 1063 queue = sk->sk_user_data; 1064 if (likely(queue && sk_stream_is_writeable(sk))) { 1065 clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 1066 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 1067 } 1068 read_unlock_bh(&sk->sk_callback_lock); 1069 } 1070 1071 static void nvme_tcp_state_change(struct sock *sk) 1072 { 1073 struct nvme_tcp_queue *queue; 1074 1075 read_lock_bh(&sk->sk_callback_lock); 1076 queue = sk->sk_user_data; 1077 if (!queue) 1078 goto done; 1079 1080 switch (sk->sk_state) { 1081 case TCP_CLOSE: 1082 case TCP_CLOSE_WAIT: 1083 case TCP_LAST_ACK: 1084 case TCP_FIN_WAIT1: 1085 case TCP_FIN_WAIT2: 1086 nvme_tcp_error_recovery(&queue->ctrl->ctrl); 1087 break; 1088 default: 1089 dev_info(queue->ctrl->ctrl.device, 1090 "queue %d socket state %d\n", 1091 nvme_tcp_queue_id(queue), sk->sk_state); 1092 } 1093 1094 queue->state_change(sk); 1095 done: 1096 read_unlock_bh(&sk->sk_callback_lock); 1097 } 1098 1099 static inline void nvme_tcp_done_send_req(struct nvme_tcp_queue *queue) 1100 { 1101 queue->request = NULL; 1102 } 1103 1104 static void nvme_tcp_fail_request(struct nvme_tcp_request *req) 1105 { 1106 if (nvme_tcp_async_req(req)) { 1107 union nvme_result res = {}; 1108 1109 nvme_complete_async_event(&req->queue->ctrl->ctrl, 1110 cpu_to_le16(NVME_SC_HOST_PATH_ERROR), &res); 1111 } else { 1112 nvme_tcp_end_request(blk_mq_rq_from_pdu(req), 1113 NVME_SC_HOST_PATH_ERROR); 1114 } 1115 } 1116 1117 static int nvme_tcp_try_send_data(struct nvme_tcp_request *req) 1118 { 1119 struct nvme_tcp_queue *queue = req->queue; 1120 int req_data_len = req->data_len; 1121 u32 h2cdata_left = req->h2cdata_left; 1122 1123 while (true) { 1124 struct bio_vec bvec; 1125 struct msghdr msg = { 1126 .msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES, 1127 }; 1128 struct page *page = nvme_tcp_req_cur_page(req); 1129 size_t offset = nvme_tcp_req_cur_offset(req); 1130 size_t len = nvme_tcp_req_cur_length(req); 1131 bool last = nvme_tcp_pdu_last_send(req, len); 1132 int req_data_sent = req->data_sent; 1133 int ret; 1134 1135 if (last && !queue->data_digest && !nvme_tcp_queue_more(queue)) 1136 msg.msg_flags |= MSG_EOR; 1137 else 1138 msg.msg_flags |= MSG_MORE; 1139 1140 if (!sendpages_ok(page, len, offset)) 1141 msg.msg_flags &= ~MSG_SPLICE_PAGES; 1142 1143 bvec_set_page(&bvec, page, len, offset); 1144 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, len); 1145 ret = sock_sendmsg(queue->sock, &msg); 1146 if (ret <= 0) 1147 return ret; 1148 1149 if (queue->data_digest) 1150 nvme_tcp_ddgst_update(queue->snd_hash, page, 1151 offset, ret); 1152 1153 /* 1154 * update the request iterator except for the last payload send 1155 * in the request where we don't want to modify it as we may 1156 * compete with the RX path completing the request. 1157 */ 1158 if (req_data_sent + ret < req_data_len) 1159 nvme_tcp_advance_req(req, ret); 1160 1161 /* fully successful last send in current PDU */ 1162 if (last && ret == len) { 1163 if (queue->data_digest) { 1164 nvme_tcp_ddgst_final(queue->snd_hash, 1165 &req->ddgst); 1166 req->state = NVME_TCP_SEND_DDGST; 1167 req->offset = 0; 1168 } else { 1169 if (h2cdata_left) 1170 nvme_tcp_setup_h2c_data_pdu(req); 1171 else 1172 nvme_tcp_done_send_req(queue); 1173 } 1174 return 1; 1175 } 1176 } 1177 return -EAGAIN; 1178 } 1179 1180 static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req) 1181 { 1182 struct nvme_tcp_queue *queue = req->queue; 1183 struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req); 1184 struct bio_vec bvec; 1185 struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES, }; 1186 bool inline_data = nvme_tcp_has_inline_data(req); 1187 u8 hdgst = nvme_tcp_hdgst_len(queue); 1188 int len = sizeof(*pdu) + hdgst - req->offset; 1189 int ret; 1190 1191 if (inline_data || nvme_tcp_queue_more(queue)) 1192 msg.msg_flags |= MSG_MORE; 1193 else 1194 msg.msg_flags |= MSG_EOR; 1195 1196 if (queue->hdr_digest && !req->offset) 1197 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu)); 1198 1199 bvec_set_virt(&bvec, (void *)pdu + req->offset, len); 1200 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, len); 1201 ret = sock_sendmsg(queue->sock, &msg); 1202 if (unlikely(ret <= 0)) 1203 return ret; 1204 1205 len -= ret; 1206 if (!len) { 1207 if (inline_data) { 1208 req->state = NVME_TCP_SEND_DATA; 1209 if (queue->data_digest) 1210 crypto_ahash_init(queue->snd_hash); 1211 } else { 1212 nvme_tcp_done_send_req(queue); 1213 } 1214 return 1; 1215 } 1216 req->offset += ret; 1217 1218 return -EAGAIN; 1219 } 1220 1221 static int nvme_tcp_try_send_data_pdu(struct nvme_tcp_request *req) 1222 { 1223 struct nvme_tcp_queue *queue = req->queue; 1224 struct nvme_tcp_data_pdu *pdu = nvme_tcp_req_data_pdu(req); 1225 struct bio_vec bvec; 1226 struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_MORE, }; 1227 u8 hdgst = nvme_tcp_hdgst_len(queue); 1228 int len = sizeof(*pdu) - req->offset + hdgst; 1229 int ret; 1230 1231 if (queue->hdr_digest && !req->offset) 1232 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu)); 1233 1234 if (!req->h2cdata_left) 1235 msg.msg_flags |= MSG_SPLICE_PAGES; 1236 1237 bvec_set_virt(&bvec, (void *)pdu + req->offset, len); 1238 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, len); 1239 ret = sock_sendmsg(queue->sock, &msg); 1240 if (unlikely(ret <= 0)) 1241 return ret; 1242 1243 len -= ret; 1244 if (!len) { 1245 req->state = NVME_TCP_SEND_DATA; 1246 if (queue->data_digest) 1247 crypto_ahash_init(queue->snd_hash); 1248 return 1; 1249 } 1250 req->offset += ret; 1251 1252 return -EAGAIN; 1253 } 1254 1255 static int nvme_tcp_try_send_ddgst(struct nvme_tcp_request *req) 1256 { 1257 struct nvme_tcp_queue *queue = req->queue; 1258 size_t offset = req->offset; 1259 u32 h2cdata_left = req->h2cdata_left; 1260 int ret; 1261 struct msghdr msg = { .msg_flags = MSG_DONTWAIT }; 1262 struct kvec iov = { 1263 .iov_base = (u8 *)&req->ddgst + req->offset, 1264 .iov_len = NVME_TCP_DIGEST_LENGTH - req->offset 1265 }; 1266 1267 if (nvme_tcp_queue_more(queue)) 1268 msg.msg_flags |= MSG_MORE; 1269 else 1270 msg.msg_flags |= MSG_EOR; 1271 1272 ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len); 1273 if (unlikely(ret <= 0)) 1274 return ret; 1275 1276 if (offset + ret == NVME_TCP_DIGEST_LENGTH) { 1277 if (h2cdata_left) 1278 nvme_tcp_setup_h2c_data_pdu(req); 1279 else 1280 nvme_tcp_done_send_req(queue); 1281 return 1; 1282 } 1283 1284 req->offset += ret; 1285 return -EAGAIN; 1286 } 1287 1288 static int nvme_tcp_try_send(struct nvme_tcp_queue *queue) 1289 { 1290 struct nvme_tcp_request *req; 1291 unsigned int noreclaim_flag; 1292 int ret = 1; 1293 1294 if (!queue->request) { 1295 queue->request = nvme_tcp_fetch_request(queue); 1296 if (!queue->request) 1297 return 0; 1298 } 1299 req = queue->request; 1300 1301 noreclaim_flag = memalloc_noreclaim_save(); 1302 if (req->state == NVME_TCP_SEND_CMD_PDU) { 1303 ret = nvme_tcp_try_send_cmd_pdu(req); 1304 if (ret <= 0) 1305 goto done; 1306 if (!nvme_tcp_has_inline_data(req)) 1307 goto out; 1308 } 1309 1310 if (req->state == NVME_TCP_SEND_H2C_PDU) { 1311 ret = nvme_tcp_try_send_data_pdu(req); 1312 if (ret <= 0) 1313 goto done; 1314 } 1315 1316 if (req->state == NVME_TCP_SEND_DATA) { 1317 ret = nvme_tcp_try_send_data(req); 1318 if (ret <= 0) 1319 goto done; 1320 } 1321 1322 if (req->state == NVME_TCP_SEND_DDGST) 1323 ret = nvme_tcp_try_send_ddgst(req); 1324 done: 1325 if (ret == -EAGAIN) { 1326 ret = 0; 1327 } else if (ret < 0) { 1328 dev_err(queue->ctrl->ctrl.device, 1329 "failed to send request %d\n", ret); 1330 nvme_tcp_fail_request(queue->request); 1331 nvme_tcp_done_send_req(queue); 1332 } 1333 out: 1334 memalloc_noreclaim_restore(noreclaim_flag); 1335 return ret; 1336 } 1337 1338 static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue) 1339 { 1340 struct socket *sock = queue->sock; 1341 struct sock *sk = sock->sk; 1342 read_descriptor_t rd_desc; 1343 int consumed; 1344 1345 rd_desc.arg.data = queue; 1346 rd_desc.count = 1; 1347 lock_sock(sk); 1348 queue->nr_cqe = 0; 1349 consumed = sock->ops->read_sock(sk, &rd_desc, nvme_tcp_recv_skb); 1350 release_sock(sk); 1351 return consumed; 1352 } 1353 1354 static void nvme_tcp_io_work(struct work_struct *w) 1355 { 1356 struct nvme_tcp_queue *queue = 1357 container_of(w, struct nvme_tcp_queue, io_work); 1358 unsigned long deadline = jiffies + msecs_to_jiffies(1); 1359 1360 do { 1361 bool pending = false; 1362 int result; 1363 1364 if (mutex_trylock(&queue->send_mutex)) { 1365 result = nvme_tcp_try_send(queue); 1366 mutex_unlock(&queue->send_mutex); 1367 if (result > 0) 1368 pending = true; 1369 else if (unlikely(result < 0)) 1370 break; 1371 } 1372 1373 result = nvme_tcp_try_recv(queue); 1374 if (result > 0) 1375 pending = true; 1376 else if (unlikely(result < 0)) 1377 return; 1378 1379 if (!pending || !queue->rd_enabled) 1380 return; 1381 1382 } while (!time_after(jiffies, deadline)); /* quota is exhausted */ 1383 1384 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 1385 } 1386 1387 static void nvme_tcp_free_crypto(struct nvme_tcp_queue *queue) 1388 { 1389 struct crypto_ahash *tfm = crypto_ahash_reqtfm(queue->rcv_hash); 1390 1391 ahash_request_free(queue->rcv_hash); 1392 ahash_request_free(queue->snd_hash); 1393 crypto_free_ahash(tfm); 1394 } 1395 1396 static int nvme_tcp_alloc_crypto(struct nvme_tcp_queue *queue) 1397 { 1398 struct crypto_ahash *tfm; 1399 1400 tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC); 1401 if (IS_ERR(tfm)) 1402 return PTR_ERR(tfm); 1403 1404 queue->snd_hash = ahash_request_alloc(tfm, GFP_KERNEL); 1405 if (!queue->snd_hash) 1406 goto free_tfm; 1407 ahash_request_set_callback(queue->snd_hash, 0, NULL, NULL); 1408 1409 queue->rcv_hash = ahash_request_alloc(tfm, GFP_KERNEL); 1410 if (!queue->rcv_hash) 1411 goto free_snd_hash; 1412 ahash_request_set_callback(queue->rcv_hash, 0, NULL, NULL); 1413 1414 return 0; 1415 free_snd_hash: 1416 ahash_request_free(queue->snd_hash); 1417 free_tfm: 1418 crypto_free_ahash(tfm); 1419 return -ENOMEM; 1420 } 1421 1422 static void nvme_tcp_free_async_req(struct nvme_tcp_ctrl *ctrl) 1423 { 1424 struct nvme_tcp_request *async = &ctrl->async_req; 1425 1426 page_frag_free(async->pdu); 1427 } 1428 1429 static int nvme_tcp_alloc_async_req(struct nvme_tcp_ctrl *ctrl) 1430 { 1431 struct nvme_tcp_queue *queue = &ctrl->queues[0]; 1432 struct nvme_tcp_request *async = &ctrl->async_req; 1433 u8 hdgst = nvme_tcp_hdgst_len(queue); 1434 1435 async->pdu = page_frag_alloc(&queue->pf_cache, 1436 sizeof(struct nvme_tcp_cmd_pdu) + hdgst, 1437 GFP_KERNEL | __GFP_ZERO); 1438 if (!async->pdu) 1439 return -ENOMEM; 1440 1441 async->queue = &ctrl->queues[0]; 1442 return 0; 1443 } 1444 1445 static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid) 1446 { 1447 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1448 struct nvme_tcp_queue *queue = &ctrl->queues[qid]; 1449 unsigned int noreclaim_flag; 1450 1451 if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags)) 1452 return; 1453 1454 if (queue->hdr_digest || queue->data_digest) 1455 nvme_tcp_free_crypto(queue); 1456 1457 page_frag_cache_drain(&queue->pf_cache); 1458 1459 noreclaim_flag = memalloc_noreclaim_save(); 1460 /* ->sock will be released by fput() */ 1461 fput(queue->sock->file); 1462 queue->sock = NULL; 1463 memalloc_noreclaim_restore(noreclaim_flag); 1464 1465 kfree(queue->pdu); 1466 mutex_destroy(&queue->send_mutex); 1467 mutex_destroy(&queue->queue_lock); 1468 } 1469 1470 static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue) 1471 { 1472 struct nvme_tcp_icreq_pdu *icreq; 1473 struct nvme_tcp_icresp_pdu *icresp; 1474 char cbuf[CMSG_LEN(sizeof(char))] = {}; 1475 u8 ctype; 1476 struct msghdr msg = {}; 1477 struct kvec iov; 1478 bool ctrl_hdgst, ctrl_ddgst; 1479 u32 maxh2cdata; 1480 int ret; 1481 1482 icreq = kzalloc(sizeof(*icreq), GFP_KERNEL); 1483 if (!icreq) 1484 return -ENOMEM; 1485 1486 icresp = kzalloc(sizeof(*icresp), GFP_KERNEL); 1487 if (!icresp) { 1488 ret = -ENOMEM; 1489 goto free_icreq; 1490 } 1491 1492 icreq->hdr.type = nvme_tcp_icreq; 1493 icreq->hdr.hlen = sizeof(*icreq); 1494 icreq->hdr.pdo = 0; 1495 icreq->hdr.plen = cpu_to_le32(icreq->hdr.hlen); 1496 icreq->pfv = cpu_to_le16(NVME_TCP_PFV_1_0); 1497 icreq->maxr2t = 0; /* single inflight r2t supported */ 1498 icreq->hpda = 0; /* no alignment constraint */ 1499 if (queue->hdr_digest) 1500 icreq->digest |= NVME_TCP_HDR_DIGEST_ENABLE; 1501 if (queue->data_digest) 1502 icreq->digest |= NVME_TCP_DATA_DIGEST_ENABLE; 1503 1504 iov.iov_base = icreq; 1505 iov.iov_len = sizeof(*icreq); 1506 ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len); 1507 if (ret < 0) { 1508 pr_warn("queue %d: failed to send icreq, error %d\n", 1509 nvme_tcp_queue_id(queue), ret); 1510 goto free_icresp; 1511 } 1512 1513 memset(&msg, 0, sizeof(msg)); 1514 iov.iov_base = icresp; 1515 iov.iov_len = sizeof(*icresp); 1516 if (nvme_tcp_queue_tls(queue)) { 1517 msg.msg_control = cbuf; 1518 msg.msg_controllen = sizeof(cbuf); 1519 } 1520 msg.msg_flags = MSG_WAITALL; 1521 ret = kernel_recvmsg(queue->sock, &msg, &iov, 1, 1522 iov.iov_len, msg.msg_flags); 1523 if (ret >= 0 && ret < sizeof(*icresp)) 1524 ret = -ECONNRESET; 1525 if (ret < 0) { 1526 pr_warn("queue %d: failed to receive icresp, error %d\n", 1527 nvme_tcp_queue_id(queue), ret); 1528 goto free_icresp; 1529 } 1530 ret = -ENOTCONN; 1531 if (nvme_tcp_queue_tls(queue)) { 1532 ctype = tls_get_record_type(queue->sock->sk, 1533 (struct cmsghdr *)cbuf); 1534 if (ctype != TLS_RECORD_TYPE_DATA) { 1535 pr_err("queue %d: unhandled TLS record %d\n", 1536 nvme_tcp_queue_id(queue), ctype); 1537 goto free_icresp; 1538 } 1539 } 1540 ret = -EINVAL; 1541 if (icresp->hdr.type != nvme_tcp_icresp) { 1542 pr_err("queue %d: bad type returned %d\n", 1543 nvme_tcp_queue_id(queue), icresp->hdr.type); 1544 goto free_icresp; 1545 } 1546 1547 if (le32_to_cpu(icresp->hdr.plen) != sizeof(*icresp)) { 1548 pr_err("queue %d: bad pdu length returned %d\n", 1549 nvme_tcp_queue_id(queue), icresp->hdr.plen); 1550 goto free_icresp; 1551 } 1552 1553 if (icresp->pfv != NVME_TCP_PFV_1_0) { 1554 pr_err("queue %d: bad pfv returned %d\n", 1555 nvme_tcp_queue_id(queue), icresp->pfv); 1556 goto free_icresp; 1557 } 1558 1559 ctrl_ddgst = !!(icresp->digest & NVME_TCP_DATA_DIGEST_ENABLE); 1560 if ((queue->data_digest && !ctrl_ddgst) || 1561 (!queue->data_digest && ctrl_ddgst)) { 1562 pr_err("queue %d: data digest mismatch host: %s ctrl: %s\n", 1563 nvme_tcp_queue_id(queue), 1564 queue->data_digest ? "enabled" : "disabled", 1565 ctrl_ddgst ? "enabled" : "disabled"); 1566 goto free_icresp; 1567 } 1568 1569 ctrl_hdgst = !!(icresp->digest & NVME_TCP_HDR_DIGEST_ENABLE); 1570 if ((queue->hdr_digest && !ctrl_hdgst) || 1571 (!queue->hdr_digest && ctrl_hdgst)) { 1572 pr_err("queue %d: header digest mismatch host: %s ctrl: %s\n", 1573 nvme_tcp_queue_id(queue), 1574 queue->hdr_digest ? "enabled" : "disabled", 1575 ctrl_hdgst ? "enabled" : "disabled"); 1576 goto free_icresp; 1577 } 1578 1579 if (icresp->cpda != 0) { 1580 pr_err("queue %d: unsupported cpda returned %d\n", 1581 nvme_tcp_queue_id(queue), icresp->cpda); 1582 goto free_icresp; 1583 } 1584 1585 maxh2cdata = le32_to_cpu(icresp->maxdata); 1586 if ((maxh2cdata % 4) || (maxh2cdata < NVME_TCP_MIN_MAXH2CDATA)) { 1587 pr_err("queue %d: invalid maxh2cdata returned %u\n", 1588 nvme_tcp_queue_id(queue), maxh2cdata); 1589 goto free_icresp; 1590 } 1591 queue->maxh2cdata = maxh2cdata; 1592 1593 ret = 0; 1594 free_icresp: 1595 kfree(icresp); 1596 free_icreq: 1597 kfree(icreq); 1598 return ret; 1599 } 1600 1601 static bool nvme_tcp_admin_queue(struct nvme_tcp_queue *queue) 1602 { 1603 return nvme_tcp_queue_id(queue) == 0; 1604 } 1605 1606 static bool nvme_tcp_default_queue(struct nvme_tcp_queue *queue) 1607 { 1608 struct nvme_tcp_ctrl *ctrl = queue->ctrl; 1609 int qid = nvme_tcp_queue_id(queue); 1610 1611 return !nvme_tcp_admin_queue(queue) && 1612 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT]; 1613 } 1614 1615 static bool nvme_tcp_read_queue(struct nvme_tcp_queue *queue) 1616 { 1617 struct nvme_tcp_ctrl *ctrl = queue->ctrl; 1618 int qid = nvme_tcp_queue_id(queue); 1619 1620 return !nvme_tcp_admin_queue(queue) && 1621 !nvme_tcp_default_queue(queue) && 1622 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] + 1623 ctrl->io_queues[HCTX_TYPE_READ]; 1624 } 1625 1626 static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue) 1627 { 1628 struct nvme_tcp_ctrl *ctrl = queue->ctrl; 1629 int qid = nvme_tcp_queue_id(queue); 1630 1631 return !nvme_tcp_admin_queue(queue) && 1632 !nvme_tcp_default_queue(queue) && 1633 !nvme_tcp_read_queue(queue) && 1634 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] + 1635 ctrl->io_queues[HCTX_TYPE_READ] + 1636 ctrl->io_queues[HCTX_TYPE_POLL]; 1637 } 1638 1639 /* 1640 * Track the number of queues assigned to each cpu using a global per-cpu 1641 * counter and select the least used cpu from the mq_map. Our goal is to spread 1642 * different controllers I/O threads across different cpu cores. 1643 * 1644 * Note that the accounting is not 100% perfect, but we don't need to be, we're 1645 * simply putting our best effort to select the best candidate cpu core that we 1646 * find at any given point. 1647 */ 1648 static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue) 1649 { 1650 struct nvme_tcp_ctrl *ctrl = queue->ctrl; 1651 struct blk_mq_tag_set *set = &ctrl->tag_set; 1652 int qid = nvme_tcp_queue_id(queue) - 1; 1653 unsigned int *mq_map = NULL; 1654 int cpu, min_queues = INT_MAX, io_cpu; 1655 1656 if (wq_unbound) 1657 goto out; 1658 1659 if (nvme_tcp_default_queue(queue)) 1660 mq_map = set->map[HCTX_TYPE_DEFAULT].mq_map; 1661 else if (nvme_tcp_read_queue(queue)) 1662 mq_map = set->map[HCTX_TYPE_READ].mq_map; 1663 else if (nvme_tcp_poll_queue(queue)) 1664 mq_map = set->map[HCTX_TYPE_POLL].mq_map; 1665 1666 if (WARN_ON(!mq_map)) 1667 goto out; 1668 1669 /* Search for the least used cpu from the mq_map */ 1670 io_cpu = WORK_CPU_UNBOUND; 1671 for_each_online_cpu(cpu) { 1672 int num_queues = atomic_read(&nvme_tcp_cpu_queues[cpu]); 1673 1674 if (mq_map[cpu] != qid) 1675 continue; 1676 if (num_queues < min_queues) { 1677 io_cpu = cpu; 1678 min_queues = num_queues; 1679 } 1680 } 1681 if (io_cpu != WORK_CPU_UNBOUND) { 1682 queue->io_cpu = io_cpu; 1683 atomic_inc(&nvme_tcp_cpu_queues[io_cpu]); 1684 set_bit(NVME_TCP_Q_IO_CPU_SET, &queue->flags); 1685 } 1686 out: 1687 dev_dbg(ctrl->ctrl.device, "queue %d: using cpu %d\n", 1688 qid, queue->io_cpu); 1689 } 1690 1691 static void nvme_tcp_tls_done(void *data, int status, key_serial_t pskid) 1692 { 1693 struct nvme_tcp_queue *queue = data; 1694 struct nvme_tcp_ctrl *ctrl = queue->ctrl; 1695 int qid = nvme_tcp_queue_id(queue); 1696 struct key *tls_key; 1697 1698 dev_dbg(ctrl->ctrl.device, "queue %d: TLS handshake done, key %x, status %d\n", 1699 qid, pskid, status); 1700 1701 if (status) { 1702 queue->tls_err = -status; 1703 goto out_complete; 1704 } 1705 1706 tls_key = nvme_tls_key_lookup(pskid); 1707 if (IS_ERR(tls_key)) { 1708 dev_warn(ctrl->ctrl.device, "queue %d: Invalid key %x\n", 1709 qid, pskid); 1710 queue->tls_err = -ENOKEY; 1711 } else { 1712 queue->tls_enabled = true; 1713 if (qid == 0) 1714 ctrl->ctrl.tls_pskid = key_serial(tls_key); 1715 key_put(tls_key); 1716 queue->tls_err = 0; 1717 } 1718 1719 out_complete: 1720 complete(&queue->tls_complete); 1721 } 1722 1723 static int nvme_tcp_start_tls(struct nvme_ctrl *nctrl, 1724 struct nvme_tcp_queue *queue, 1725 key_serial_t pskid) 1726 { 1727 int qid = nvme_tcp_queue_id(queue); 1728 int ret; 1729 struct tls_handshake_args args; 1730 unsigned long tmo = tls_handshake_timeout * HZ; 1731 key_serial_t keyring = nvme_keyring_id(); 1732 1733 dev_dbg(nctrl->device, "queue %d: start TLS with key %x\n", 1734 qid, pskid); 1735 memset(&args, 0, sizeof(args)); 1736 args.ta_sock = queue->sock; 1737 args.ta_done = nvme_tcp_tls_done; 1738 args.ta_data = queue; 1739 args.ta_my_peerids[0] = pskid; 1740 args.ta_num_peerids = 1; 1741 if (nctrl->opts->keyring) 1742 keyring = key_serial(nctrl->opts->keyring); 1743 args.ta_keyring = keyring; 1744 args.ta_timeout_ms = tls_handshake_timeout * 1000; 1745 queue->tls_err = -EOPNOTSUPP; 1746 init_completion(&queue->tls_complete); 1747 ret = tls_client_hello_psk(&args, GFP_KERNEL); 1748 if (ret) { 1749 dev_err(nctrl->device, "queue %d: failed to start TLS: %d\n", 1750 qid, ret); 1751 return ret; 1752 } 1753 ret = wait_for_completion_interruptible_timeout(&queue->tls_complete, tmo); 1754 if (ret <= 0) { 1755 if (ret == 0) 1756 ret = -ETIMEDOUT; 1757 1758 dev_err(nctrl->device, 1759 "queue %d: TLS handshake failed, error %d\n", 1760 qid, ret); 1761 tls_handshake_cancel(queue->sock->sk); 1762 } else { 1763 dev_dbg(nctrl->device, 1764 "queue %d: TLS handshake complete, error %d\n", 1765 qid, queue->tls_err); 1766 ret = queue->tls_err; 1767 } 1768 return ret; 1769 } 1770 1771 static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid, 1772 key_serial_t pskid) 1773 { 1774 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1775 struct nvme_tcp_queue *queue = &ctrl->queues[qid]; 1776 int ret, rcv_pdu_size; 1777 struct file *sock_file; 1778 1779 mutex_init(&queue->queue_lock); 1780 queue->ctrl = ctrl; 1781 init_llist_head(&queue->req_list); 1782 INIT_LIST_HEAD(&queue->send_list); 1783 mutex_init(&queue->send_mutex); 1784 INIT_WORK(&queue->io_work, nvme_tcp_io_work); 1785 1786 if (qid > 0) 1787 queue->cmnd_capsule_len = nctrl->ioccsz * 16; 1788 else 1789 queue->cmnd_capsule_len = sizeof(struct nvme_command) + 1790 NVME_TCP_ADMIN_CCSZ; 1791 1792 ret = sock_create_kern(current->nsproxy->net_ns, 1793 ctrl->addr.ss_family, SOCK_STREAM, 1794 IPPROTO_TCP, &queue->sock); 1795 if (ret) { 1796 dev_err(nctrl->device, 1797 "failed to create socket: %d\n", ret); 1798 goto err_destroy_mutex; 1799 } 1800 1801 sock_file = sock_alloc_file(queue->sock, O_CLOEXEC, NULL); 1802 if (IS_ERR(sock_file)) { 1803 ret = PTR_ERR(sock_file); 1804 goto err_destroy_mutex; 1805 } 1806 nvme_tcp_reclassify_socket(queue->sock); 1807 1808 /* Single syn retry */ 1809 tcp_sock_set_syncnt(queue->sock->sk, 1); 1810 1811 /* Set TCP no delay */ 1812 tcp_sock_set_nodelay(queue->sock->sk); 1813 1814 /* 1815 * Cleanup whatever is sitting in the TCP transmit queue on socket 1816 * close. This is done to prevent stale data from being sent should 1817 * the network connection be restored before TCP times out. 1818 */ 1819 sock_no_linger(queue->sock->sk); 1820 1821 if (so_priority > 0) 1822 sock_set_priority(queue->sock->sk, so_priority); 1823 1824 /* Set socket type of service */ 1825 if (nctrl->opts->tos >= 0) 1826 ip_sock_set_tos(queue->sock->sk, nctrl->opts->tos); 1827 1828 /* Set 10 seconds timeout for icresp recvmsg */ 1829 queue->sock->sk->sk_rcvtimeo = 10 * HZ; 1830 1831 queue->sock->sk->sk_allocation = GFP_ATOMIC; 1832 queue->sock->sk->sk_use_task_frag = false; 1833 queue->io_cpu = WORK_CPU_UNBOUND; 1834 queue->request = NULL; 1835 queue->data_remaining = 0; 1836 queue->ddgst_remaining = 0; 1837 queue->pdu_remaining = 0; 1838 queue->pdu_offset = 0; 1839 sk_set_memalloc(queue->sock->sk); 1840 1841 if (nctrl->opts->mask & NVMF_OPT_HOST_TRADDR) { 1842 ret = kernel_bind(queue->sock, (struct sockaddr *)&ctrl->src_addr, 1843 sizeof(ctrl->src_addr)); 1844 if (ret) { 1845 dev_err(nctrl->device, 1846 "failed to bind queue %d socket %d\n", 1847 qid, ret); 1848 goto err_sock; 1849 } 1850 } 1851 1852 if (nctrl->opts->mask & NVMF_OPT_HOST_IFACE) { 1853 char *iface = nctrl->opts->host_iface; 1854 sockptr_t optval = KERNEL_SOCKPTR(iface); 1855 1856 ret = sock_setsockopt(queue->sock, SOL_SOCKET, SO_BINDTODEVICE, 1857 optval, strlen(iface)); 1858 if (ret) { 1859 dev_err(nctrl->device, 1860 "failed to bind to interface %s queue %d err %d\n", 1861 iface, qid, ret); 1862 goto err_sock; 1863 } 1864 } 1865 1866 queue->hdr_digest = nctrl->opts->hdr_digest; 1867 queue->data_digest = nctrl->opts->data_digest; 1868 if (queue->hdr_digest || queue->data_digest) { 1869 ret = nvme_tcp_alloc_crypto(queue); 1870 if (ret) { 1871 dev_err(nctrl->device, 1872 "failed to allocate queue %d crypto\n", qid); 1873 goto err_sock; 1874 } 1875 } 1876 1877 rcv_pdu_size = sizeof(struct nvme_tcp_rsp_pdu) + 1878 nvme_tcp_hdgst_len(queue); 1879 queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL); 1880 if (!queue->pdu) { 1881 ret = -ENOMEM; 1882 goto err_crypto; 1883 } 1884 1885 dev_dbg(nctrl->device, "connecting queue %d\n", 1886 nvme_tcp_queue_id(queue)); 1887 1888 ret = kernel_connect(queue->sock, (struct sockaddr *)&ctrl->addr, 1889 sizeof(ctrl->addr), 0); 1890 if (ret) { 1891 dev_err(nctrl->device, 1892 "failed to connect socket: %d\n", ret); 1893 goto err_rcv_pdu; 1894 } 1895 1896 /* If PSKs are configured try to start TLS */ 1897 if (nvme_tcp_tls_configured(nctrl) && pskid) { 1898 ret = nvme_tcp_start_tls(nctrl, queue, pskid); 1899 if (ret) 1900 goto err_init_connect; 1901 } 1902 1903 ret = nvme_tcp_init_connection(queue); 1904 if (ret) 1905 goto err_init_connect; 1906 1907 set_bit(NVME_TCP_Q_ALLOCATED, &queue->flags); 1908 1909 return 0; 1910 1911 err_init_connect: 1912 kernel_sock_shutdown(queue->sock, SHUT_RDWR); 1913 err_rcv_pdu: 1914 kfree(queue->pdu); 1915 err_crypto: 1916 if (queue->hdr_digest || queue->data_digest) 1917 nvme_tcp_free_crypto(queue); 1918 err_sock: 1919 /* ->sock will be released by fput() */ 1920 fput(queue->sock->file); 1921 queue->sock = NULL; 1922 err_destroy_mutex: 1923 mutex_destroy(&queue->send_mutex); 1924 mutex_destroy(&queue->queue_lock); 1925 return ret; 1926 } 1927 1928 static void nvme_tcp_restore_sock_ops(struct nvme_tcp_queue *queue) 1929 { 1930 struct socket *sock = queue->sock; 1931 1932 write_lock_bh(&sock->sk->sk_callback_lock); 1933 sock->sk->sk_user_data = NULL; 1934 sock->sk->sk_data_ready = queue->data_ready; 1935 sock->sk->sk_state_change = queue->state_change; 1936 sock->sk->sk_write_space = queue->write_space; 1937 write_unlock_bh(&sock->sk->sk_callback_lock); 1938 } 1939 1940 static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue) 1941 { 1942 kernel_sock_shutdown(queue->sock, SHUT_RDWR); 1943 nvme_tcp_restore_sock_ops(queue); 1944 cancel_work_sync(&queue->io_work); 1945 } 1946 1947 static void nvme_tcp_stop_queue(struct nvme_ctrl *nctrl, int qid) 1948 { 1949 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1950 struct nvme_tcp_queue *queue = &ctrl->queues[qid]; 1951 1952 if (!test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags)) 1953 return; 1954 1955 if (test_and_clear_bit(NVME_TCP_Q_IO_CPU_SET, &queue->flags)) 1956 atomic_dec(&nvme_tcp_cpu_queues[queue->io_cpu]); 1957 1958 mutex_lock(&queue->queue_lock); 1959 if (test_and_clear_bit(NVME_TCP_Q_LIVE, &queue->flags)) 1960 __nvme_tcp_stop_queue(queue); 1961 /* Stopping the queue will disable TLS */ 1962 queue->tls_enabled = false; 1963 mutex_unlock(&queue->queue_lock); 1964 } 1965 1966 static void nvme_tcp_setup_sock_ops(struct nvme_tcp_queue *queue) 1967 { 1968 write_lock_bh(&queue->sock->sk->sk_callback_lock); 1969 queue->sock->sk->sk_user_data = queue; 1970 queue->state_change = queue->sock->sk->sk_state_change; 1971 queue->data_ready = queue->sock->sk->sk_data_ready; 1972 queue->write_space = queue->sock->sk->sk_write_space; 1973 queue->sock->sk->sk_data_ready = nvme_tcp_data_ready; 1974 queue->sock->sk->sk_state_change = nvme_tcp_state_change; 1975 queue->sock->sk->sk_write_space = nvme_tcp_write_space; 1976 #ifdef CONFIG_NET_RX_BUSY_POLL 1977 queue->sock->sk->sk_ll_usec = 1; 1978 #endif 1979 write_unlock_bh(&queue->sock->sk->sk_callback_lock); 1980 } 1981 1982 static int nvme_tcp_start_queue(struct nvme_ctrl *nctrl, int idx) 1983 { 1984 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1985 struct nvme_tcp_queue *queue = &ctrl->queues[idx]; 1986 int ret; 1987 1988 queue->rd_enabled = true; 1989 nvme_tcp_init_recv_ctx(queue); 1990 nvme_tcp_setup_sock_ops(queue); 1991 1992 if (idx) { 1993 nvme_tcp_set_queue_io_cpu(queue); 1994 ret = nvmf_connect_io_queue(nctrl, idx); 1995 } else 1996 ret = nvmf_connect_admin_queue(nctrl); 1997 1998 if (!ret) { 1999 set_bit(NVME_TCP_Q_LIVE, &queue->flags); 2000 } else { 2001 if (test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags)) 2002 __nvme_tcp_stop_queue(queue); 2003 dev_err(nctrl->device, 2004 "failed to connect queue: %d ret=%d\n", idx, ret); 2005 } 2006 return ret; 2007 } 2008 2009 static void nvme_tcp_free_admin_queue(struct nvme_ctrl *ctrl) 2010 { 2011 if (to_tcp_ctrl(ctrl)->async_req.pdu) { 2012 cancel_work_sync(&ctrl->async_event_work); 2013 nvme_tcp_free_async_req(to_tcp_ctrl(ctrl)); 2014 to_tcp_ctrl(ctrl)->async_req.pdu = NULL; 2015 } 2016 2017 nvme_tcp_free_queue(ctrl, 0); 2018 } 2019 2020 static void nvme_tcp_free_io_queues(struct nvme_ctrl *ctrl) 2021 { 2022 int i; 2023 2024 for (i = 1; i < ctrl->queue_count; i++) 2025 nvme_tcp_free_queue(ctrl, i); 2026 } 2027 2028 static void nvme_tcp_stop_io_queues(struct nvme_ctrl *ctrl) 2029 { 2030 int i; 2031 2032 for (i = 1; i < ctrl->queue_count; i++) 2033 nvme_tcp_stop_queue(ctrl, i); 2034 } 2035 2036 static int nvme_tcp_start_io_queues(struct nvme_ctrl *ctrl, 2037 int first, int last) 2038 { 2039 int i, ret; 2040 2041 for (i = first; i < last; i++) { 2042 ret = nvme_tcp_start_queue(ctrl, i); 2043 if (ret) 2044 goto out_stop_queues; 2045 } 2046 2047 return 0; 2048 2049 out_stop_queues: 2050 for (i--; i >= first; i--) 2051 nvme_tcp_stop_queue(ctrl, i); 2052 return ret; 2053 } 2054 2055 static int nvme_tcp_alloc_admin_queue(struct nvme_ctrl *ctrl) 2056 { 2057 int ret; 2058 key_serial_t pskid = 0; 2059 2060 if (nvme_tcp_tls_configured(ctrl)) { 2061 if (ctrl->opts->tls_key) 2062 pskid = key_serial(ctrl->opts->tls_key); 2063 else if (ctrl->opts->tls) { 2064 pskid = nvme_tls_psk_default(ctrl->opts->keyring, 2065 ctrl->opts->host->nqn, 2066 ctrl->opts->subsysnqn); 2067 if (!pskid) { 2068 dev_err(ctrl->device, "no valid PSK found\n"); 2069 return -ENOKEY; 2070 } 2071 } 2072 } 2073 2074 ret = nvme_tcp_alloc_queue(ctrl, 0, pskid); 2075 if (ret) 2076 return ret; 2077 2078 ret = nvme_tcp_alloc_async_req(to_tcp_ctrl(ctrl)); 2079 if (ret) 2080 goto out_free_queue; 2081 2082 return 0; 2083 2084 out_free_queue: 2085 nvme_tcp_free_queue(ctrl, 0); 2086 return ret; 2087 } 2088 2089 static int __nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl) 2090 { 2091 int i, ret; 2092 2093 if (nvme_tcp_tls_configured(ctrl)) { 2094 if (ctrl->opts->concat) { 2095 /* 2096 * The generated PSK is stored in the 2097 * fabric options 2098 */ 2099 if (!ctrl->opts->tls_key) { 2100 dev_err(ctrl->device, "no PSK generated\n"); 2101 return -ENOKEY; 2102 } 2103 if (ctrl->tls_pskid && 2104 ctrl->tls_pskid != key_serial(ctrl->opts->tls_key)) { 2105 dev_err(ctrl->device, "Stale PSK id %08x\n", ctrl->tls_pskid); 2106 ctrl->tls_pskid = 0; 2107 } 2108 } else if (!ctrl->tls_pskid) { 2109 dev_err(ctrl->device, "no PSK negotiated\n"); 2110 return -ENOKEY; 2111 } 2112 } 2113 2114 for (i = 1; i < ctrl->queue_count; i++) { 2115 ret = nvme_tcp_alloc_queue(ctrl, i, 2116 ctrl->tls_pskid); 2117 if (ret) 2118 goto out_free_queues; 2119 } 2120 2121 return 0; 2122 2123 out_free_queues: 2124 for (i--; i >= 1; i--) 2125 nvme_tcp_free_queue(ctrl, i); 2126 2127 return ret; 2128 } 2129 2130 static int nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl) 2131 { 2132 unsigned int nr_io_queues; 2133 int ret; 2134 2135 nr_io_queues = nvmf_nr_io_queues(ctrl->opts); 2136 ret = nvme_set_queue_count(ctrl, &nr_io_queues); 2137 if (ret) 2138 return ret; 2139 2140 if (nr_io_queues == 0) { 2141 dev_err(ctrl->device, 2142 "unable to set any I/O queues\n"); 2143 return -ENOMEM; 2144 } 2145 2146 ctrl->queue_count = nr_io_queues + 1; 2147 dev_info(ctrl->device, 2148 "creating %d I/O queues.\n", nr_io_queues); 2149 2150 nvmf_set_io_queues(ctrl->opts, nr_io_queues, 2151 to_tcp_ctrl(ctrl)->io_queues); 2152 return __nvme_tcp_alloc_io_queues(ctrl); 2153 } 2154 2155 static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new) 2156 { 2157 int ret, nr_queues; 2158 2159 ret = nvme_tcp_alloc_io_queues(ctrl); 2160 if (ret) 2161 return ret; 2162 2163 if (new) { 2164 ret = nvme_alloc_io_tag_set(ctrl, &to_tcp_ctrl(ctrl)->tag_set, 2165 &nvme_tcp_mq_ops, 2166 ctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2, 2167 sizeof(struct nvme_tcp_request)); 2168 if (ret) 2169 goto out_free_io_queues; 2170 } 2171 2172 /* 2173 * Only start IO queues for which we have allocated the tagset 2174 * and limitted it to the available queues. On reconnects, the 2175 * queue number might have changed. 2176 */ 2177 nr_queues = min(ctrl->tagset->nr_hw_queues + 1, ctrl->queue_count); 2178 ret = nvme_tcp_start_io_queues(ctrl, 1, nr_queues); 2179 if (ret) 2180 goto out_cleanup_connect_q; 2181 2182 if (!new) { 2183 nvme_start_freeze(ctrl); 2184 nvme_unquiesce_io_queues(ctrl); 2185 if (!nvme_wait_freeze_timeout(ctrl, NVME_IO_TIMEOUT)) { 2186 /* 2187 * If we timed out waiting for freeze we are likely to 2188 * be stuck. Fail the controller initialization just 2189 * to be safe. 2190 */ 2191 ret = -ENODEV; 2192 nvme_unfreeze(ctrl); 2193 goto out_wait_freeze_timed_out; 2194 } 2195 blk_mq_update_nr_hw_queues(ctrl->tagset, 2196 ctrl->queue_count - 1); 2197 nvme_unfreeze(ctrl); 2198 } 2199 2200 /* 2201 * If the number of queues has increased (reconnect case) 2202 * start all new queues now. 2203 */ 2204 ret = nvme_tcp_start_io_queues(ctrl, nr_queues, 2205 ctrl->tagset->nr_hw_queues + 1); 2206 if (ret) 2207 goto out_wait_freeze_timed_out; 2208 2209 return 0; 2210 2211 out_wait_freeze_timed_out: 2212 nvme_quiesce_io_queues(ctrl); 2213 nvme_sync_io_queues(ctrl); 2214 nvme_tcp_stop_io_queues(ctrl); 2215 out_cleanup_connect_q: 2216 nvme_cancel_tagset(ctrl); 2217 if (new) 2218 nvme_remove_io_tag_set(ctrl); 2219 out_free_io_queues: 2220 nvme_tcp_free_io_queues(ctrl); 2221 return ret; 2222 } 2223 2224 static int nvme_tcp_configure_admin_queue(struct nvme_ctrl *ctrl, bool new) 2225 { 2226 int error; 2227 2228 error = nvme_tcp_alloc_admin_queue(ctrl); 2229 if (error) 2230 return error; 2231 2232 if (new) { 2233 error = nvme_alloc_admin_tag_set(ctrl, 2234 &to_tcp_ctrl(ctrl)->admin_tag_set, 2235 &nvme_tcp_admin_mq_ops, 2236 sizeof(struct nvme_tcp_request)); 2237 if (error) 2238 goto out_free_queue; 2239 } 2240 2241 error = nvme_tcp_start_queue(ctrl, 0); 2242 if (error) 2243 goto out_cleanup_tagset; 2244 2245 error = nvme_enable_ctrl(ctrl); 2246 if (error) 2247 goto out_stop_queue; 2248 2249 nvme_unquiesce_admin_queue(ctrl); 2250 2251 error = nvme_init_ctrl_finish(ctrl, false); 2252 if (error) 2253 goto out_quiesce_queue; 2254 2255 return 0; 2256 2257 out_quiesce_queue: 2258 nvme_quiesce_admin_queue(ctrl); 2259 blk_sync_queue(ctrl->admin_q); 2260 out_stop_queue: 2261 nvme_tcp_stop_queue(ctrl, 0); 2262 nvme_cancel_admin_tagset(ctrl); 2263 out_cleanup_tagset: 2264 if (new) 2265 nvme_remove_admin_tag_set(ctrl); 2266 out_free_queue: 2267 nvme_tcp_free_admin_queue(ctrl); 2268 return error; 2269 } 2270 2271 static void nvme_tcp_teardown_admin_queue(struct nvme_ctrl *ctrl, 2272 bool remove) 2273 { 2274 nvme_quiesce_admin_queue(ctrl); 2275 blk_sync_queue(ctrl->admin_q); 2276 nvme_tcp_stop_queue(ctrl, 0); 2277 nvme_cancel_admin_tagset(ctrl); 2278 if (remove) { 2279 nvme_unquiesce_admin_queue(ctrl); 2280 nvme_remove_admin_tag_set(ctrl); 2281 } 2282 nvme_tcp_free_admin_queue(ctrl); 2283 if (ctrl->tls_pskid) { 2284 dev_dbg(ctrl->device, "Wipe negotiated TLS_PSK %08x\n", 2285 ctrl->tls_pskid); 2286 ctrl->tls_pskid = 0; 2287 } 2288 } 2289 2290 static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl, 2291 bool remove) 2292 { 2293 if (ctrl->queue_count <= 1) 2294 return; 2295 nvme_quiesce_io_queues(ctrl); 2296 nvme_sync_io_queues(ctrl); 2297 nvme_tcp_stop_io_queues(ctrl); 2298 nvme_cancel_tagset(ctrl); 2299 if (remove) { 2300 nvme_unquiesce_io_queues(ctrl); 2301 nvme_remove_io_tag_set(ctrl); 2302 } 2303 nvme_tcp_free_io_queues(ctrl); 2304 } 2305 2306 static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl, 2307 int status) 2308 { 2309 enum nvme_ctrl_state state = nvme_ctrl_state(ctrl); 2310 2311 /* If we are resetting/deleting then do nothing */ 2312 if (state != NVME_CTRL_CONNECTING) { 2313 WARN_ON_ONCE(state == NVME_CTRL_NEW || state == NVME_CTRL_LIVE); 2314 return; 2315 } 2316 2317 if (nvmf_should_reconnect(ctrl, status)) { 2318 dev_info(ctrl->device, "Reconnecting in %d seconds...\n", 2319 ctrl->opts->reconnect_delay); 2320 queue_delayed_work(nvme_wq, &to_tcp_ctrl(ctrl)->connect_work, 2321 ctrl->opts->reconnect_delay * HZ); 2322 } else { 2323 dev_info(ctrl->device, "Removing controller (%d)...\n", 2324 status); 2325 nvme_delete_ctrl(ctrl); 2326 } 2327 } 2328 2329 /* 2330 * The TLS key is set by secure concatenation after negotiation has been 2331 * completed on the admin queue. We need to revoke the key when: 2332 * - concatenation is enabled (otherwise it's a static key set by the user) 2333 * and 2334 * - the generated key is present in ctrl->tls_key (otherwise there's nothing 2335 * to revoke) 2336 * and 2337 * - a valid PSK key ID has been set in ctrl->tls_pskid (otherwise TLS 2338 * negotiation has not run). 2339 * 2340 * We cannot always revoke the key as nvme_tcp_alloc_admin_queue() is called 2341 * twice during secure concatenation, once on a 'normal' connection to run the 2342 * DH-HMAC-CHAP negotiation (which generates the key, so it _must not_ be set), 2343 * and once after the negotiation (which uses the key, so it _must_ be set). 2344 */ 2345 static bool nvme_tcp_key_revoke_needed(struct nvme_ctrl *ctrl) 2346 { 2347 return ctrl->opts->concat && ctrl->opts->tls_key && ctrl->tls_pskid; 2348 } 2349 2350 static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new) 2351 { 2352 struct nvmf_ctrl_options *opts = ctrl->opts; 2353 int ret; 2354 2355 ret = nvme_tcp_configure_admin_queue(ctrl, new); 2356 if (ret) 2357 return ret; 2358 2359 if (ctrl->opts && ctrl->opts->concat && !ctrl->tls_pskid) { 2360 /* See comments for nvme_tcp_key_revoke_needed() */ 2361 dev_dbg(ctrl->device, "restart admin queue for secure concatenation\n"); 2362 nvme_stop_keep_alive(ctrl); 2363 nvme_tcp_teardown_admin_queue(ctrl, false); 2364 ret = nvme_tcp_configure_admin_queue(ctrl, false); 2365 if (ret) 2366 return ret; 2367 } 2368 2369 if (ctrl->icdoff) { 2370 ret = -EOPNOTSUPP; 2371 dev_err(ctrl->device, "icdoff is not supported!\n"); 2372 goto destroy_admin; 2373 } 2374 2375 if (!nvme_ctrl_sgl_supported(ctrl)) { 2376 ret = -EOPNOTSUPP; 2377 dev_err(ctrl->device, "Mandatory sgls are not supported!\n"); 2378 goto destroy_admin; 2379 } 2380 2381 if (opts->queue_size > ctrl->sqsize + 1) 2382 dev_warn(ctrl->device, 2383 "queue_size %zu > ctrl sqsize %u, clamping down\n", 2384 opts->queue_size, ctrl->sqsize + 1); 2385 2386 if (ctrl->sqsize + 1 > ctrl->maxcmd) { 2387 dev_warn(ctrl->device, 2388 "sqsize %u > ctrl maxcmd %u, clamping down\n", 2389 ctrl->sqsize + 1, ctrl->maxcmd); 2390 ctrl->sqsize = ctrl->maxcmd - 1; 2391 } 2392 2393 if (ctrl->queue_count > 1) { 2394 ret = nvme_tcp_configure_io_queues(ctrl, new); 2395 if (ret) 2396 goto destroy_admin; 2397 } 2398 2399 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE)) { 2400 /* 2401 * state change failure is ok if we started ctrl delete, 2402 * unless we're during creation of a new controller to 2403 * avoid races with teardown flow. 2404 */ 2405 enum nvme_ctrl_state state = nvme_ctrl_state(ctrl); 2406 2407 WARN_ON_ONCE(state != NVME_CTRL_DELETING && 2408 state != NVME_CTRL_DELETING_NOIO); 2409 WARN_ON_ONCE(new); 2410 ret = -EINVAL; 2411 goto destroy_io; 2412 } 2413 2414 nvme_start_ctrl(ctrl); 2415 return 0; 2416 2417 destroy_io: 2418 if (ctrl->queue_count > 1) { 2419 nvme_quiesce_io_queues(ctrl); 2420 nvme_sync_io_queues(ctrl); 2421 nvme_tcp_stop_io_queues(ctrl); 2422 nvme_cancel_tagset(ctrl); 2423 if (new) 2424 nvme_remove_io_tag_set(ctrl); 2425 nvme_tcp_free_io_queues(ctrl); 2426 } 2427 destroy_admin: 2428 nvme_stop_keep_alive(ctrl); 2429 nvme_tcp_teardown_admin_queue(ctrl, new); 2430 return ret; 2431 } 2432 2433 static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work) 2434 { 2435 struct nvme_tcp_ctrl *tcp_ctrl = container_of(to_delayed_work(work), 2436 struct nvme_tcp_ctrl, connect_work); 2437 struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl; 2438 int ret; 2439 2440 ++ctrl->nr_reconnects; 2441 2442 ret = nvme_tcp_setup_ctrl(ctrl, false); 2443 if (ret) 2444 goto requeue; 2445 2446 dev_info(ctrl->device, "Successfully reconnected (attempt %d/%d)\n", 2447 ctrl->nr_reconnects, ctrl->opts->max_reconnects); 2448 2449 ctrl->nr_reconnects = 0; 2450 2451 return; 2452 2453 requeue: 2454 dev_info(ctrl->device, "Failed reconnect attempt %d/%d\n", 2455 ctrl->nr_reconnects, ctrl->opts->max_reconnects); 2456 nvme_tcp_reconnect_or_remove(ctrl, ret); 2457 } 2458 2459 static void nvme_tcp_error_recovery_work(struct work_struct *work) 2460 { 2461 struct nvme_tcp_ctrl *tcp_ctrl = container_of(work, 2462 struct nvme_tcp_ctrl, err_work); 2463 struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl; 2464 2465 if (nvme_tcp_key_revoke_needed(ctrl)) 2466 nvme_auth_revoke_tls_key(ctrl); 2467 nvme_stop_keep_alive(ctrl); 2468 flush_work(&ctrl->async_event_work); 2469 nvme_tcp_teardown_io_queues(ctrl, false); 2470 /* unquiesce to fail fast pending requests */ 2471 nvme_unquiesce_io_queues(ctrl); 2472 nvme_tcp_teardown_admin_queue(ctrl, false); 2473 nvme_unquiesce_admin_queue(ctrl); 2474 nvme_auth_stop(ctrl); 2475 2476 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) { 2477 /* state change failure is ok if we started ctrl delete */ 2478 enum nvme_ctrl_state state = nvme_ctrl_state(ctrl); 2479 2480 WARN_ON_ONCE(state != NVME_CTRL_DELETING && 2481 state != NVME_CTRL_DELETING_NOIO); 2482 return; 2483 } 2484 2485 nvme_tcp_reconnect_or_remove(ctrl, 0); 2486 } 2487 2488 static void nvme_tcp_teardown_ctrl(struct nvme_ctrl *ctrl, bool shutdown) 2489 { 2490 nvme_tcp_teardown_io_queues(ctrl, shutdown); 2491 nvme_quiesce_admin_queue(ctrl); 2492 nvme_disable_ctrl(ctrl, shutdown); 2493 nvme_tcp_teardown_admin_queue(ctrl, shutdown); 2494 } 2495 2496 static void nvme_tcp_delete_ctrl(struct nvme_ctrl *ctrl) 2497 { 2498 nvme_tcp_teardown_ctrl(ctrl, true); 2499 } 2500 2501 static void nvme_reset_ctrl_work(struct work_struct *work) 2502 { 2503 struct nvme_ctrl *ctrl = 2504 container_of(work, struct nvme_ctrl, reset_work); 2505 int ret; 2506 2507 if (nvme_tcp_key_revoke_needed(ctrl)) 2508 nvme_auth_revoke_tls_key(ctrl); 2509 nvme_stop_ctrl(ctrl); 2510 nvme_tcp_teardown_ctrl(ctrl, false); 2511 2512 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) { 2513 /* state change failure is ok if we started ctrl delete */ 2514 enum nvme_ctrl_state state = nvme_ctrl_state(ctrl); 2515 2516 WARN_ON_ONCE(state != NVME_CTRL_DELETING && 2517 state != NVME_CTRL_DELETING_NOIO); 2518 return; 2519 } 2520 2521 ret = nvme_tcp_setup_ctrl(ctrl, false); 2522 if (ret) 2523 goto out_fail; 2524 2525 return; 2526 2527 out_fail: 2528 ++ctrl->nr_reconnects; 2529 nvme_tcp_reconnect_or_remove(ctrl, ret); 2530 } 2531 2532 static void nvme_tcp_stop_ctrl(struct nvme_ctrl *ctrl) 2533 { 2534 flush_work(&to_tcp_ctrl(ctrl)->err_work); 2535 cancel_delayed_work_sync(&to_tcp_ctrl(ctrl)->connect_work); 2536 } 2537 2538 static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl) 2539 { 2540 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 2541 2542 if (list_empty(&ctrl->list)) 2543 goto free_ctrl; 2544 2545 mutex_lock(&nvme_tcp_ctrl_mutex); 2546 list_del(&ctrl->list); 2547 mutex_unlock(&nvme_tcp_ctrl_mutex); 2548 2549 nvmf_free_options(nctrl->opts); 2550 free_ctrl: 2551 kfree(ctrl->queues); 2552 kfree(ctrl); 2553 } 2554 2555 static void nvme_tcp_set_sg_null(struct nvme_command *c) 2556 { 2557 struct nvme_sgl_desc *sg = &c->common.dptr.sgl; 2558 2559 sg->addr = 0; 2560 sg->length = 0; 2561 sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) | 2562 NVME_SGL_FMT_TRANSPORT_A; 2563 } 2564 2565 static void nvme_tcp_set_sg_inline(struct nvme_tcp_queue *queue, 2566 struct nvme_command *c, u32 data_len) 2567 { 2568 struct nvme_sgl_desc *sg = &c->common.dptr.sgl; 2569 2570 sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff); 2571 sg->length = cpu_to_le32(data_len); 2572 sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET; 2573 } 2574 2575 static void nvme_tcp_set_sg_host_data(struct nvme_command *c, 2576 u32 data_len) 2577 { 2578 struct nvme_sgl_desc *sg = &c->common.dptr.sgl; 2579 2580 sg->addr = 0; 2581 sg->length = cpu_to_le32(data_len); 2582 sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) | 2583 NVME_SGL_FMT_TRANSPORT_A; 2584 } 2585 2586 static void nvme_tcp_submit_async_event(struct nvme_ctrl *arg) 2587 { 2588 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(arg); 2589 struct nvme_tcp_queue *queue = &ctrl->queues[0]; 2590 struct nvme_tcp_cmd_pdu *pdu = ctrl->async_req.pdu; 2591 struct nvme_command *cmd = &pdu->cmd; 2592 u8 hdgst = nvme_tcp_hdgst_len(queue); 2593 2594 memset(pdu, 0, sizeof(*pdu)); 2595 pdu->hdr.type = nvme_tcp_cmd; 2596 if (queue->hdr_digest) 2597 pdu->hdr.flags |= NVME_TCP_F_HDGST; 2598 pdu->hdr.hlen = sizeof(*pdu); 2599 pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst); 2600 2601 cmd->common.opcode = nvme_admin_async_event; 2602 cmd->common.command_id = NVME_AQ_BLK_MQ_DEPTH; 2603 cmd->common.flags |= NVME_CMD_SGL_METABUF; 2604 nvme_tcp_set_sg_null(cmd); 2605 2606 ctrl->async_req.state = NVME_TCP_SEND_CMD_PDU; 2607 ctrl->async_req.offset = 0; 2608 ctrl->async_req.curr_bio = NULL; 2609 ctrl->async_req.data_len = 0; 2610 2611 nvme_tcp_queue_request(&ctrl->async_req, true, true); 2612 } 2613 2614 static void nvme_tcp_complete_timed_out(struct request *rq) 2615 { 2616 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2617 struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl; 2618 2619 nvme_tcp_stop_queue(ctrl, nvme_tcp_queue_id(req->queue)); 2620 nvmf_complete_timed_out_request(rq); 2621 } 2622 2623 static enum blk_eh_timer_return nvme_tcp_timeout(struct request *rq) 2624 { 2625 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2626 struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl; 2627 struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req); 2628 struct nvme_command *cmd = &pdu->cmd; 2629 int qid = nvme_tcp_queue_id(req->queue); 2630 2631 dev_warn(ctrl->device, 2632 "I/O tag %d (%04x) type %d opcode %#x (%s) QID %d timeout\n", 2633 rq->tag, nvme_cid(rq), pdu->hdr.type, cmd->common.opcode, 2634 nvme_fabrics_opcode_str(qid, cmd), qid); 2635 2636 if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE) { 2637 /* 2638 * If we are resetting, connecting or deleting we should 2639 * complete immediately because we may block controller 2640 * teardown or setup sequence 2641 * - ctrl disable/shutdown fabrics requests 2642 * - connect requests 2643 * - initialization admin requests 2644 * - I/O requests that entered after unquiescing and 2645 * the controller stopped responding 2646 * 2647 * All other requests should be cancelled by the error 2648 * recovery work, so it's fine that we fail it here. 2649 */ 2650 nvme_tcp_complete_timed_out(rq); 2651 return BLK_EH_DONE; 2652 } 2653 2654 /* 2655 * LIVE state should trigger the normal error recovery which will 2656 * handle completing this request. 2657 */ 2658 nvme_tcp_error_recovery(ctrl); 2659 return BLK_EH_RESET_TIMER; 2660 } 2661 2662 static blk_status_t nvme_tcp_map_data(struct nvme_tcp_queue *queue, 2663 struct request *rq) 2664 { 2665 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2666 struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req); 2667 struct nvme_command *c = &pdu->cmd; 2668 2669 c->common.flags |= NVME_CMD_SGL_METABUF; 2670 2671 if (!blk_rq_nr_phys_segments(rq)) 2672 nvme_tcp_set_sg_null(c); 2673 else if (rq_data_dir(rq) == WRITE && 2674 req->data_len <= nvme_tcp_inline_data_size(req)) 2675 nvme_tcp_set_sg_inline(queue, c, req->data_len); 2676 else 2677 nvme_tcp_set_sg_host_data(c, req->data_len); 2678 2679 return 0; 2680 } 2681 2682 static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns, 2683 struct request *rq) 2684 { 2685 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2686 struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req); 2687 struct nvme_tcp_queue *queue = req->queue; 2688 u8 hdgst = nvme_tcp_hdgst_len(queue), ddgst = 0; 2689 blk_status_t ret; 2690 2691 ret = nvme_setup_cmd(ns, rq); 2692 if (ret) 2693 return ret; 2694 2695 req->state = NVME_TCP_SEND_CMD_PDU; 2696 req->status = cpu_to_le16(NVME_SC_SUCCESS); 2697 req->offset = 0; 2698 req->data_sent = 0; 2699 req->pdu_len = 0; 2700 req->pdu_sent = 0; 2701 req->h2cdata_left = 0; 2702 req->data_len = blk_rq_nr_phys_segments(rq) ? 2703 blk_rq_payload_bytes(rq) : 0; 2704 req->curr_bio = rq->bio; 2705 if (req->curr_bio && req->data_len) 2706 nvme_tcp_init_iter(req, rq_data_dir(rq)); 2707 2708 if (rq_data_dir(rq) == WRITE && 2709 req->data_len <= nvme_tcp_inline_data_size(req)) 2710 req->pdu_len = req->data_len; 2711 2712 pdu->hdr.type = nvme_tcp_cmd; 2713 pdu->hdr.flags = 0; 2714 if (queue->hdr_digest) 2715 pdu->hdr.flags |= NVME_TCP_F_HDGST; 2716 if (queue->data_digest && req->pdu_len) { 2717 pdu->hdr.flags |= NVME_TCP_F_DDGST; 2718 ddgst = nvme_tcp_ddgst_len(queue); 2719 } 2720 pdu->hdr.hlen = sizeof(*pdu); 2721 pdu->hdr.pdo = req->pdu_len ? pdu->hdr.hlen + hdgst : 0; 2722 pdu->hdr.plen = 2723 cpu_to_le32(pdu->hdr.hlen + hdgst + req->pdu_len + ddgst); 2724 2725 ret = nvme_tcp_map_data(queue, rq); 2726 if (unlikely(ret)) { 2727 nvme_cleanup_cmd(rq); 2728 dev_err(queue->ctrl->ctrl.device, 2729 "Failed to map data (%d)\n", ret); 2730 return ret; 2731 } 2732 2733 return 0; 2734 } 2735 2736 static void nvme_tcp_commit_rqs(struct blk_mq_hw_ctx *hctx) 2737 { 2738 struct nvme_tcp_queue *queue = hctx->driver_data; 2739 2740 if (!llist_empty(&queue->req_list)) 2741 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 2742 } 2743 2744 static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx, 2745 const struct blk_mq_queue_data *bd) 2746 { 2747 struct nvme_ns *ns = hctx->queue->queuedata; 2748 struct nvme_tcp_queue *queue = hctx->driver_data; 2749 struct request *rq = bd->rq; 2750 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2751 bool queue_ready = test_bit(NVME_TCP_Q_LIVE, &queue->flags); 2752 blk_status_t ret; 2753 2754 if (!nvme_check_ready(&queue->ctrl->ctrl, rq, queue_ready)) 2755 return nvme_fail_nonready_command(&queue->ctrl->ctrl, rq); 2756 2757 ret = nvme_tcp_setup_cmd_pdu(ns, rq); 2758 if (unlikely(ret)) 2759 return ret; 2760 2761 nvme_start_request(rq); 2762 2763 nvme_tcp_queue_request(req, true, bd->last); 2764 2765 return BLK_STS_OK; 2766 } 2767 2768 static void nvme_tcp_map_queues(struct blk_mq_tag_set *set) 2769 { 2770 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(set->driver_data); 2771 2772 nvmf_map_queues(set, &ctrl->ctrl, ctrl->io_queues); 2773 } 2774 2775 static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) 2776 { 2777 struct nvme_tcp_queue *queue = hctx->driver_data; 2778 struct sock *sk = queue->sock->sk; 2779 int ret; 2780 2781 if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags)) 2782 return 0; 2783 2784 set_bit(NVME_TCP_Q_POLLING, &queue->flags); 2785 if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(&sk->sk_receive_queue)) 2786 sk_busy_loop(sk, true); 2787 ret = nvme_tcp_try_recv(queue); 2788 clear_bit(NVME_TCP_Q_POLLING, &queue->flags); 2789 return ret < 0 ? ret : queue->nr_cqe; 2790 } 2791 2792 static int nvme_tcp_get_address(struct nvme_ctrl *ctrl, char *buf, int size) 2793 { 2794 struct nvme_tcp_queue *queue = &to_tcp_ctrl(ctrl)->queues[0]; 2795 struct sockaddr_storage src_addr; 2796 int ret, len; 2797 2798 len = nvmf_get_address(ctrl, buf, size); 2799 2800 if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags)) 2801 return len; 2802 2803 mutex_lock(&queue->queue_lock); 2804 2805 ret = kernel_getsockname(queue->sock, (struct sockaddr *)&src_addr); 2806 if (ret > 0) { 2807 if (len > 0) 2808 len--; /* strip trailing newline */ 2809 len += scnprintf(buf + len, size - len, "%ssrc_addr=%pISc\n", 2810 (len) ? "," : "", &src_addr); 2811 } 2812 2813 mutex_unlock(&queue->queue_lock); 2814 2815 return len; 2816 } 2817 2818 static const struct blk_mq_ops nvme_tcp_mq_ops = { 2819 .queue_rq = nvme_tcp_queue_rq, 2820 .commit_rqs = nvme_tcp_commit_rqs, 2821 .complete = nvme_complete_rq, 2822 .init_request = nvme_tcp_init_request, 2823 .exit_request = nvme_tcp_exit_request, 2824 .init_hctx = nvme_tcp_init_hctx, 2825 .timeout = nvme_tcp_timeout, 2826 .map_queues = nvme_tcp_map_queues, 2827 .poll = nvme_tcp_poll, 2828 }; 2829 2830 static const struct blk_mq_ops nvme_tcp_admin_mq_ops = { 2831 .queue_rq = nvme_tcp_queue_rq, 2832 .complete = nvme_complete_rq, 2833 .init_request = nvme_tcp_init_request, 2834 .exit_request = nvme_tcp_exit_request, 2835 .init_hctx = nvme_tcp_init_admin_hctx, 2836 .timeout = nvme_tcp_timeout, 2837 }; 2838 2839 static const struct nvme_ctrl_ops nvme_tcp_ctrl_ops = { 2840 .name = "tcp", 2841 .module = THIS_MODULE, 2842 .flags = NVME_F_FABRICS | NVME_F_BLOCKING, 2843 .reg_read32 = nvmf_reg_read32, 2844 .reg_read64 = nvmf_reg_read64, 2845 .reg_write32 = nvmf_reg_write32, 2846 .subsystem_reset = nvmf_subsystem_reset, 2847 .free_ctrl = nvme_tcp_free_ctrl, 2848 .submit_async_event = nvme_tcp_submit_async_event, 2849 .delete_ctrl = nvme_tcp_delete_ctrl, 2850 .get_address = nvme_tcp_get_address, 2851 .stop_ctrl = nvme_tcp_stop_ctrl, 2852 }; 2853 2854 static bool 2855 nvme_tcp_existing_controller(struct nvmf_ctrl_options *opts) 2856 { 2857 struct nvme_tcp_ctrl *ctrl; 2858 bool found = false; 2859 2860 mutex_lock(&nvme_tcp_ctrl_mutex); 2861 list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) { 2862 found = nvmf_ip_options_match(&ctrl->ctrl, opts); 2863 if (found) 2864 break; 2865 } 2866 mutex_unlock(&nvme_tcp_ctrl_mutex); 2867 2868 return found; 2869 } 2870 2871 static struct nvme_tcp_ctrl *nvme_tcp_alloc_ctrl(struct device *dev, 2872 struct nvmf_ctrl_options *opts) 2873 { 2874 struct nvme_tcp_ctrl *ctrl; 2875 int ret; 2876 2877 ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL); 2878 if (!ctrl) 2879 return ERR_PTR(-ENOMEM); 2880 2881 INIT_LIST_HEAD(&ctrl->list); 2882 ctrl->ctrl.opts = opts; 2883 ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues + 2884 opts->nr_poll_queues + 1; 2885 ctrl->ctrl.sqsize = opts->queue_size - 1; 2886 ctrl->ctrl.kato = opts->kato; 2887 2888 INIT_DELAYED_WORK(&ctrl->connect_work, 2889 nvme_tcp_reconnect_ctrl_work); 2890 INIT_WORK(&ctrl->err_work, nvme_tcp_error_recovery_work); 2891 INIT_WORK(&ctrl->ctrl.reset_work, nvme_reset_ctrl_work); 2892 2893 if (!(opts->mask & NVMF_OPT_TRSVCID)) { 2894 opts->trsvcid = 2895 kstrdup(__stringify(NVME_TCP_DISC_PORT), GFP_KERNEL); 2896 if (!opts->trsvcid) { 2897 ret = -ENOMEM; 2898 goto out_free_ctrl; 2899 } 2900 opts->mask |= NVMF_OPT_TRSVCID; 2901 } 2902 2903 ret = inet_pton_with_scope(&init_net, AF_UNSPEC, 2904 opts->traddr, opts->trsvcid, &ctrl->addr); 2905 if (ret) { 2906 pr_err("malformed address passed: %s:%s\n", 2907 opts->traddr, opts->trsvcid); 2908 goto out_free_ctrl; 2909 } 2910 2911 if (opts->mask & NVMF_OPT_HOST_TRADDR) { 2912 ret = inet_pton_with_scope(&init_net, AF_UNSPEC, 2913 opts->host_traddr, NULL, &ctrl->src_addr); 2914 if (ret) { 2915 pr_err("malformed src address passed: %s\n", 2916 opts->host_traddr); 2917 goto out_free_ctrl; 2918 } 2919 } 2920 2921 if (opts->mask & NVMF_OPT_HOST_IFACE) { 2922 if (!__dev_get_by_name(&init_net, opts->host_iface)) { 2923 pr_err("invalid interface passed: %s\n", 2924 opts->host_iface); 2925 ret = -ENODEV; 2926 goto out_free_ctrl; 2927 } 2928 } 2929 2930 if (!opts->duplicate_connect && nvme_tcp_existing_controller(opts)) { 2931 ret = -EALREADY; 2932 goto out_free_ctrl; 2933 } 2934 2935 ctrl->queues = kcalloc(ctrl->ctrl.queue_count, sizeof(*ctrl->queues), 2936 GFP_KERNEL); 2937 if (!ctrl->queues) { 2938 ret = -ENOMEM; 2939 goto out_free_ctrl; 2940 } 2941 2942 ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_tcp_ctrl_ops, 0); 2943 if (ret) 2944 goto out_kfree_queues; 2945 2946 return ctrl; 2947 out_kfree_queues: 2948 kfree(ctrl->queues); 2949 out_free_ctrl: 2950 kfree(ctrl); 2951 return ERR_PTR(ret); 2952 } 2953 2954 static struct nvme_ctrl *nvme_tcp_create_ctrl(struct device *dev, 2955 struct nvmf_ctrl_options *opts) 2956 { 2957 struct nvme_tcp_ctrl *ctrl; 2958 int ret; 2959 2960 ctrl = nvme_tcp_alloc_ctrl(dev, opts); 2961 if (IS_ERR(ctrl)) 2962 return ERR_CAST(ctrl); 2963 2964 ret = nvme_add_ctrl(&ctrl->ctrl); 2965 if (ret) 2966 goto out_put_ctrl; 2967 2968 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) { 2969 WARN_ON_ONCE(1); 2970 ret = -EINTR; 2971 goto out_uninit_ctrl; 2972 } 2973 2974 ret = nvme_tcp_setup_ctrl(&ctrl->ctrl, true); 2975 if (ret) 2976 goto out_uninit_ctrl; 2977 2978 dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISp, hostnqn: %s\n", 2979 nvmf_ctrl_subsysnqn(&ctrl->ctrl), &ctrl->addr, opts->host->nqn); 2980 2981 mutex_lock(&nvme_tcp_ctrl_mutex); 2982 list_add_tail(&ctrl->list, &nvme_tcp_ctrl_list); 2983 mutex_unlock(&nvme_tcp_ctrl_mutex); 2984 2985 return &ctrl->ctrl; 2986 2987 out_uninit_ctrl: 2988 nvme_uninit_ctrl(&ctrl->ctrl); 2989 out_put_ctrl: 2990 nvme_put_ctrl(&ctrl->ctrl); 2991 if (ret > 0) 2992 ret = -EIO; 2993 return ERR_PTR(ret); 2994 } 2995 2996 static struct nvmf_transport_ops nvme_tcp_transport = { 2997 .name = "tcp", 2998 .module = THIS_MODULE, 2999 .required_opts = NVMF_OPT_TRADDR, 3000 .allowed_opts = NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY | 3001 NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO | 3002 NVMF_OPT_HDR_DIGEST | NVMF_OPT_DATA_DIGEST | 3003 NVMF_OPT_NR_WRITE_QUEUES | NVMF_OPT_NR_POLL_QUEUES | 3004 NVMF_OPT_TOS | NVMF_OPT_HOST_IFACE | NVMF_OPT_TLS | 3005 NVMF_OPT_KEYRING | NVMF_OPT_TLS_KEY | NVMF_OPT_CONCAT, 3006 .create_ctrl = nvme_tcp_create_ctrl, 3007 }; 3008 3009 static int __init nvme_tcp_init_module(void) 3010 { 3011 unsigned int wq_flags = WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_SYSFS; 3012 int cpu; 3013 3014 BUILD_BUG_ON(sizeof(struct nvme_tcp_hdr) != 8); 3015 BUILD_BUG_ON(sizeof(struct nvme_tcp_cmd_pdu) != 72); 3016 BUILD_BUG_ON(sizeof(struct nvme_tcp_data_pdu) != 24); 3017 BUILD_BUG_ON(sizeof(struct nvme_tcp_rsp_pdu) != 24); 3018 BUILD_BUG_ON(sizeof(struct nvme_tcp_r2t_pdu) != 24); 3019 BUILD_BUG_ON(sizeof(struct nvme_tcp_icreq_pdu) != 128); 3020 BUILD_BUG_ON(sizeof(struct nvme_tcp_icresp_pdu) != 128); 3021 BUILD_BUG_ON(sizeof(struct nvme_tcp_term_pdu) != 24); 3022 3023 if (wq_unbound) 3024 wq_flags |= WQ_UNBOUND; 3025 3026 nvme_tcp_wq = alloc_workqueue("nvme_tcp_wq", wq_flags, 0); 3027 if (!nvme_tcp_wq) 3028 return -ENOMEM; 3029 3030 for_each_possible_cpu(cpu) 3031 atomic_set(&nvme_tcp_cpu_queues[cpu], 0); 3032 3033 nvmf_register_transport(&nvme_tcp_transport); 3034 return 0; 3035 } 3036 3037 static void __exit nvme_tcp_cleanup_module(void) 3038 { 3039 struct nvme_tcp_ctrl *ctrl; 3040 3041 nvmf_unregister_transport(&nvme_tcp_transport); 3042 3043 mutex_lock(&nvme_tcp_ctrl_mutex); 3044 list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) 3045 nvme_delete_ctrl(&ctrl->ctrl); 3046 mutex_unlock(&nvme_tcp_ctrl_mutex); 3047 flush_workqueue(nvme_delete_wq); 3048 3049 destroy_workqueue(nvme_tcp_wq); 3050 } 3051 3052 module_init(nvme_tcp_init_module); 3053 module_exit(nvme_tcp_cleanup_module); 3054 3055 MODULE_DESCRIPTION("NVMe host TCP transport driver"); 3056 MODULE_LICENSE("GPL v2"); 3057