1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * NVMe over Fabrics TCP host. 4 * Copyright (c) 2018 Lightbits Labs. All rights reserved. 5 */ 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 #include <linux/module.h> 8 #include <linux/init.h> 9 #include <linux/slab.h> 10 #include <linux/err.h> 11 #include <linux/key.h> 12 #include <linux/nvme-tcp.h> 13 #include <linux/nvme-keyring.h> 14 #include <net/sock.h> 15 #include <net/tcp.h> 16 #include <net/tls.h> 17 #include <net/tls_prot.h> 18 #include <net/handshake.h> 19 #include <linux/blk-mq.h> 20 #include <crypto/hash.h> 21 #include <net/busy_poll.h> 22 #include <trace/events/sock.h> 23 24 #include "nvme.h" 25 #include "fabrics.h" 26 27 struct nvme_tcp_queue; 28 29 /* Define the socket priority to use for connections were it is desirable 30 * that the NIC consider performing optimized packet processing or filtering. 31 * A non-zero value being sufficient to indicate general consideration of any 32 * possible optimization. Making it a module param allows for alternative 33 * values that may be unique for some NIC implementations. 34 */ 35 static int so_priority; 36 module_param(so_priority, int, 0644); 37 MODULE_PARM_DESC(so_priority, "nvme tcp socket optimize priority"); 38 39 /* 40 * TLS handshake timeout 41 */ 42 static int tls_handshake_timeout = 10; 43 #ifdef CONFIG_NVME_TCP_TLS 44 module_param(tls_handshake_timeout, int, 0644); 45 MODULE_PARM_DESC(tls_handshake_timeout, 46 "nvme TLS handshake timeout in seconds (default 10)"); 47 #endif 48 49 #ifdef CONFIG_DEBUG_LOCK_ALLOC 50 /* lockdep can detect a circular dependency of the form 51 * sk_lock -> mmap_lock (page fault) -> fs locks -> sk_lock 52 * because dependencies are tracked for both nvme-tcp and user contexts. Using 53 * a separate class prevents lockdep from conflating nvme-tcp socket use with 54 * user-space socket API use. 55 */ 56 static struct lock_class_key nvme_tcp_sk_key[2]; 57 static struct lock_class_key nvme_tcp_slock_key[2]; 58 59 static void nvme_tcp_reclassify_socket(struct socket *sock) 60 { 61 struct sock *sk = sock->sk; 62 63 if (WARN_ON_ONCE(!sock_allow_reclassification(sk))) 64 return; 65 66 switch (sk->sk_family) { 67 case AF_INET: 68 sock_lock_init_class_and_name(sk, "slock-AF_INET-NVME", 69 &nvme_tcp_slock_key[0], 70 "sk_lock-AF_INET-NVME", 71 &nvme_tcp_sk_key[0]); 72 break; 73 case AF_INET6: 74 sock_lock_init_class_and_name(sk, "slock-AF_INET6-NVME", 75 &nvme_tcp_slock_key[1], 76 "sk_lock-AF_INET6-NVME", 77 &nvme_tcp_sk_key[1]); 78 break; 79 default: 80 WARN_ON_ONCE(1); 81 } 82 } 83 #else 84 static void nvme_tcp_reclassify_socket(struct socket *sock) { } 85 #endif 86 87 enum nvme_tcp_send_state { 88 NVME_TCP_SEND_CMD_PDU = 0, 89 NVME_TCP_SEND_H2C_PDU, 90 NVME_TCP_SEND_DATA, 91 NVME_TCP_SEND_DDGST, 92 }; 93 94 struct nvme_tcp_request { 95 struct nvme_request req; 96 void *pdu; 97 struct nvme_tcp_queue *queue; 98 u32 data_len; 99 u32 pdu_len; 100 u32 pdu_sent; 101 u32 h2cdata_left; 102 u32 h2cdata_offset; 103 u16 ttag; 104 __le16 status; 105 struct list_head entry; 106 struct llist_node lentry; 107 __le32 ddgst; 108 109 struct bio *curr_bio; 110 struct iov_iter iter; 111 112 /* send state */ 113 size_t offset; 114 size_t data_sent; 115 enum nvme_tcp_send_state state; 116 }; 117 118 enum nvme_tcp_queue_flags { 119 NVME_TCP_Q_ALLOCATED = 0, 120 NVME_TCP_Q_LIVE = 1, 121 NVME_TCP_Q_POLLING = 2, 122 }; 123 124 enum nvme_tcp_recv_state { 125 NVME_TCP_RECV_PDU = 0, 126 NVME_TCP_RECV_DATA, 127 NVME_TCP_RECV_DDGST, 128 }; 129 130 struct nvme_tcp_ctrl; 131 struct nvme_tcp_queue { 132 struct socket *sock; 133 struct work_struct io_work; 134 int io_cpu; 135 136 struct mutex queue_lock; 137 struct mutex send_mutex; 138 struct llist_head req_list; 139 struct list_head send_list; 140 141 /* recv state */ 142 void *pdu; 143 int pdu_remaining; 144 int pdu_offset; 145 size_t data_remaining; 146 size_t ddgst_remaining; 147 unsigned int nr_cqe; 148 149 /* send state */ 150 struct nvme_tcp_request *request; 151 152 u32 maxh2cdata; 153 size_t cmnd_capsule_len; 154 struct nvme_tcp_ctrl *ctrl; 155 unsigned long flags; 156 bool rd_enabled; 157 158 bool hdr_digest; 159 bool data_digest; 160 struct ahash_request *rcv_hash; 161 struct ahash_request *snd_hash; 162 __le32 exp_ddgst; 163 __le32 recv_ddgst; 164 struct completion tls_complete; 165 int tls_err; 166 struct page_frag_cache pf_cache; 167 168 void (*state_change)(struct sock *); 169 void (*data_ready)(struct sock *); 170 void (*write_space)(struct sock *); 171 }; 172 173 struct nvme_tcp_ctrl { 174 /* read only in the hot path */ 175 struct nvme_tcp_queue *queues; 176 struct blk_mq_tag_set tag_set; 177 178 /* other member variables */ 179 struct list_head list; 180 struct blk_mq_tag_set admin_tag_set; 181 struct sockaddr_storage addr; 182 struct sockaddr_storage src_addr; 183 struct nvme_ctrl ctrl; 184 185 struct work_struct err_work; 186 struct delayed_work connect_work; 187 struct nvme_tcp_request async_req; 188 u32 io_queues[HCTX_MAX_TYPES]; 189 }; 190 191 static LIST_HEAD(nvme_tcp_ctrl_list); 192 static DEFINE_MUTEX(nvme_tcp_ctrl_mutex); 193 static struct workqueue_struct *nvme_tcp_wq; 194 static const struct blk_mq_ops nvme_tcp_mq_ops; 195 static const struct blk_mq_ops nvme_tcp_admin_mq_ops; 196 static int nvme_tcp_try_send(struct nvme_tcp_queue *queue); 197 198 static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl) 199 { 200 return container_of(ctrl, struct nvme_tcp_ctrl, ctrl); 201 } 202 203 static inline int nvme_tcp_queue_id(struct nvme_tcp_queue *queue) 204 { 205 return queue - queue->ctrl->queues; 206 } 207 208 static inline bool nvme_tcp_tls(struct nvme_ctrl *ctrl) 209 { 210 if (!IS_ENABLED(CONFIG_NVME_TCP_TLS)) 211 return 0; 212 213 return ctrl->opts->tls; 214 } 215 216 static inline struct blk_mq_tags *nvme_tcp_tagset(struct nvme_tcp_queue *queue) 217 { 218 u32 queue_idx = nvme_tcp_queue_id(queue); 219 220 if (queue_idx == 0) 221 return queue->ctrl->admin_tag_set.tags[queue_idx]; 222 return queue->ctrl->tag_set.tags[queue_idx - 1]; 223 } 224 225 static inline u8 nvme_tcp_hdgst_len(struct nvme_tcp_queue *queue) 226 { 227 return queue->hdr_digest ? NVME_TCP_DIGEST_LENGTH : 0; 228 } 229 230 static inline u8 nvme_tcp_ddgst_len(struct nvme_tcp_queue *queue) 231 { 232 return queue->data_digest ? NVME_TCP_DIGEST_LENGTH : 0; 233 } 234 235 static inline void *nvme_tcp_req_cmd_pdu(struct nvme_tcp_request *req) 236 { 237 return req->pdu; 238 } 239 240 static inline void *nvme_tcp_req_data_pdu(struct nvme_tcp_request *req) 241 { 242 /* use the pdu space in the back for the data pdu */ 243 return req->pdu + sizeof(struct nvme_tcp_cmd_pdu) - 244 sizeof(struct nvme_tcp_data_pdu); 245 } 246 247 static inline size_t nvme_tcp_inline_data_size(struct nvme_tcp_request *req) 248 { 249 if (nvme_is_fabrics(req->req.cmd)) 250 return NVME_TCP_ADMIN_CCSZ; 251 return req->queue->cmnd_capsule_len - sizeof(struct nvme_command); 252 } 253 254 static inline bool nvme_tcp_async_req(struct nvme_tcp_request *req) 255 { 256 return req == &req->queue->ctrl->async_req; 257 } 258 259 static inline bool nvme_tcp_has_inline_data(struct nvme_tcp_request *req) 260 { 261 struct request *rq; 262 263 if (unlikely(nvme_tcp_async_req(req))) 264 return false; /* async events don't have a request */ 265 266 rq = blk_mq_rq_from_pdu(req); 267 268 return rq_data_dir(rq) == WRITE && req->data_len && 269 req->data_len <= nvme_tcp_inline_data_size(req); 270 } 271 272 static inline struct page *nvme_tcp_req_cur_page(struct nvme_tcp_request *req) 273 { 274 return req->iter.bvec->bv_page; 275 } 276 277 static inline size_t nvme_tcp_req_cur_offset(struct nvme_tcp_request *req) 278 { 279 return req->iter.bvec->bv_offset + req->iter.iov_offset; 280 } 281 282 static inline size_t nvme_tcp_req_cur_length(struct nvme_tcp_request *req) 283 { 284 return min_t(size_t, iov_iter_single_seg_count(&req->iter), 285 req->pdu_len - req->pdu_sent); 286 } 287 288 static inline size_t nvme_tcp_pdu_data_left(struct nvme_tcp_request *req) 289 { 290 return rq_data_dir(blk_mq_rq_from_pdu(req)) == WRITE ? 291 req->pdu_len - req->pdu_sent : 0; 292 } 293 294 static inline size_t nvme_tcp_pdu_last_send(struct nvme_tcp_request *req, 295 int len) 296 { 297 return nvme_tcp_pdu_data_left(req) <= len; 298 } 299 300 static void nvme_tcp_init_iter(struct nvme_tcp_request *req, 301 unsigned int dir) 302 { 303 struct request *rq = blk_mq_rq_from_pdu(req); 304 struct bio_vec *vec; 305 unsigned int size; 306 int nr_bvec; 307 size_t offset; 308 309 if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) { 310 vec = &rq->special_vec; 311 nr_bvec = 1; 312 size = blk_rq_payload_bytes(rq); 313 offset = 0; 314 } else { 315 struct bio *bio = req->curr_bio; 316 struct bvec_iter bi; 317 struct bio_vec bv; 318 319 vec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter); 320 nr_bvec = 0; 321 bio_for_each_bvec(bv, bio, bi) { 322 nr_bvec++; 323 } 324 size = bio->bi_iter.bi_size; 325 offset = bio->bi_iter.bi_bvec_done; 326 } 327 328 iov_iter_bvec(&req->iter, dir, vec, nr_bvec, size); 329 req->iter.iov_offset = offset; 330 } 331 332 static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req, 333 int len) 334 { 335 req->data_sent += len; 336 req->pdu_sent += len; 337 iov_iter_advance(&req->iter, len); 338 if (!iov_iter_count(&req->iter) && 339 req->data_sent < req->data_len) { 340 req->curr_bio = req->curr_bio->bi_next; 341 nvme_tcp_init_iter(req, ITER_SOURCE); 342 } 343 } 344 345 static inline void nvme_tcp_send_all(struct nvme_tcp_queue *queue) 346 { 347 int ret; 348 349 /* drain the send queue as much as we can... */ 350 do { 351 ret = nvme_tcp_try_send(queue); 352 } while (ret > 0); 353 } 354 355 static inline bool nvme_tcp_queue_more(struct nvme_tcp_queue *queue) 356 { 357 return !list_empty(&queue->send_list) || 358 !llist_empty(&queue->req_list); 359 } 360 361 static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req, 362 bool sync, bool last) 363 { 364 struct nvme_tcp_queue *queue = req->queue; 365 bool empty; 366 367 empty = llist_add(&req->lentry, &queue->req_list) && 368 list_empty(&queue->send_list) && !queue->request; 369 370 /* 371 * if we're the first on the send_list and we can try to send 372 * directly, otherwise queue io_work. Also, only do that if we 373 * are on the same cpu, so we don't introduce contention. 374 */ 375 if (queue->io_cpu == raw_smp_processor_id() && 376 sync && empty && mutex_trylock(&queue->send_mutex)) { 377 nvme_tcp_send_all(queue); 378 mutex_unlock(&queue->send_mutex); 379 } 380 381 if (last && nvme_tcp_queue_more(queue)) 382 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 383 } 384 385 static void nvme_tcp_process_req_list(struct nvme_tcp_queue *queue) 386 { 387 struct nvme_tcp_request *req; 388 struct llist_node *node; 389 390 for (node = llist_del_all(&queue->req_list); node; node = node->next) { 391 req = llist_entry(node, struct nvme_tcp_request, lentry); 392 list_add(&req->entry, &queue->send_list); 393 } 394 } 395 396 static inline struct nvme_tcp_request * 397 nvme_tcp_fetch_request(struct nvme_tcp_queue *queue) 398 { 399 struct nvme_tcp_request *req; 400 401 req = list_first_entry_or_null(&queue->send_list, 402 struct nvme_tcp_request, entry); 403 if (!req) { 404 nvme_tcp_process_req_list(queue); 405 req = list_first_entry_or_null(&queue->send_list, 406 struct nvme_tcp_request, entry); 407 if (unlikely(!req)) 408 return NULL; 409 } 410 411 list_del(&req->entry); 412 return req; 413 } 414 415 static inline void nvme_tcp_ddgst_final(struct ahash_request *hash, 416 __le32 *dgst) 417 { 418 ahash_request_set_crypt(hash, NULL, (u8 *)dgst, 0); 419 crypto_ahash_final(hash); 420 } 421 422 static inline void nvme_tcp_ddgst_update(struct ahash_request *hash, 423 struct page *page, off_t off, size_t len) 424 { 425 struct scatterlist sg; 426 427 sg_init_table(&sg, 1); 428 sg_set_page(&sg, page, len, off); 429 ahash_request_set_crypt(hash, &sg, NULL, len); 430 crypto_ahash_update(hash); 431 } 432 433 static inline void nvme_tcp_hdgst(struct ahash_request *hash, 434 void *pdu, size_t len) 435 { 436 struct scatterlist sg; 437 438 sg_init_one(&sg, pdu, len); 439 ahash_request_set_crypt(hash, &sg, pdu + len, len); 440 crypto_ahash_digest(hash); 441 } 442 443 static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue, 444 void *pdu, size_t pdu_len) 445 { 446 struct nvme_tcp_hdr *hdr = pdu; 447 __le32 recv_digest; 448 __le32 exp_digest; 449 450 if (unlikely(!(hdr->flags & NVME_TCP_F_HDGST))) { 451 dev_err(queue->ctrl->ctrl.device, 452 "queue %d: header digest flag is cleared\n", 453 nvme_tcp_queue_id(queue)); 454 return -EPROTO; 455 } 456 457 recv_digest = *(__le32 *)(pdu + hdr->hlen); 458 nvme_tcp_hdgst(queue->rcv_hash, pdu, pdu_len); 459 exp_digest = *(__le32 *)(pdu + hdr->hlen); 460 if (recv_digest != exp_digest) { 461 dev_err(queue->ctrl->ctrl.device, 462 "header digest error: recv %#x expected %#x\n", 463 le32_to_cpu(recv_digest), le32_to_cpu(exp_digest)); 464 return -EIO; 465 } 466 467 return 0; 468 } 469 470 static int nvme_tcp_check_ddgst(struct nvme_tcp_queue *queue, void *pdu) 471 { 472 struct nvme_tcp_hdr *hdr = pdu; 473 u8 digest_len = nvme_tcp_hdgst_len(queue); 474 u32 len; 475 476 len = le32_to_cpu(hdr->plen) - hdr->hlen - 477 ((hdr->flags & NVME_TCP_F_HDGST) ? digest_len : 0); 478 479 if (unlikely(len && !(hdr->flags & NVME_TCP_F_DDGST))) { 480 dev_err(queue->ctrl->ctrl.device, 481 "queue %d: data digest flag is cleared\n", 482 nvme_tcp_queue_id(queue)); 483 return -EPROTO; 484 } 485 crypto_ahash_init(queue->rcv_hash); 486 487 return 0; 488 } 489 490 static void nvme_tcp_exit_request(struct blk_mq_tag_set *set, 491 struct request *rq, unsigned int hctx_idx) 492 { 493 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 494 495 page_frag_free(req->pdu); 496 } 497 498 static int nvme_tcp_init_request(struct blk_mq_tag_set *set, 499 struct request *rq, unsigned int hctx_idx, 500 unsigned int numa_node) 501 { 502 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(set->driver_data); 503 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 504 struct nvme_tcp_cmd_pdu *pdu; 505 int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0; 506 struct nvme_tcp_queue *queue = &ctrl->queues[queue_idx]; 507 u8 hdgst = nvme_tcp_hdgst_len(queue); 508 509 req->pdu = page_frag_alloc(&queue->pf_cache, 510 sizeof(struct nvme_tcp_cmd_pdu) + hdgst, 511 GFP_KERNEL | __GFP_ZERO); 512 if (!req->pdu) 513 return -ENOMEM; 514 515 pdu = req->pdu; 516 req->queue = queue; 517 nvme_req(rq)->ctrl = &ctrl->ctrl; 518 nvme_req(rq)->cmd = &pdu->cmd; 519 520 return 0; 521 } 522 523 static int nvme_tcp_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, 524 unsigned int hctx_idx) 525 { 526 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(data); 527 struct nvme_tcp_queue *queue = &ctrl->queues[hctx_idx + 1]; 528 529 hctx->driver_data = queue; 530 return 0; 531 } 532 533 static int nvme_tcp_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data, 534 unsigned int hctx_idx) 535 { 536 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(data); 537 struct nvme_tcp_queue *queue = &ctrl->queues[0]; 538 539 hctx->driver_data = queue; 540 return 0; 541 } 542 543 static enum nvme_tcp_recv_state 544 nvme_tcp_recv_state(struct nvme_tcp_queue *queue) 545 { 546 return (queue->pdu_remaining) ? NVME_TCP_RECV_PDU : 547 (queue->ddgst_remaining) ? NVME_TCP_RECV_DDGST : 548 NVME_TCP_RECV_DATA; 549 } 550 551 static void nvme_tcp_init_recv_ctx(struct nvme_tcp_queue *queue) 552 { 553 queue->pdu_remaining = sizeof(struct nvme_tcp_rsp_pdu) + 554 nvme_tcp_hdgst_len(queue); 555 queue->pdu_offset = 0; 556 queue->data_remaining = -1; 557 queue->ddgst_remaining = 0; 558 } 559 560 static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl) 561 { 562 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) 563 return; 564 565 dev_warn(ctrl->device, "starting error recovery\n"); 566 queue_work(nvme_reset_wq, &to_tcp_ctrl(ctrl)->err_work); 567 } 568 569 static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue, 570 struct nvme_completion *cqe) 571 { 572 struct nvme_tcp_request *req; 573 struct request *rq; 574 575 rq = nvme_find_rq(nvme_tcp_tagset(queue), cqe->command_id); 576 if (!rq) { 577 dev_err(queue->ctrl->ctrl.device, 578 "got bad cqe.command_id %#x on queue %d\n", 579 cqe->command_id, nvme_tcp_queue_id(queue)); 580 nvme_tcp_error_recovery(&queue->ctrl->ctrl); 581 return -EINVAL; 582 } 583 584 req = blk_mq_rq_to_pdu(rq); 585 if (req->status == cpu_to_le16(NVME_SC_SUCCESS)) 586 req->status = cqe->status; 587 588 if (!nvme_try_complete_req(rq, req->status, cqe->result)) 589 nvme_complete_rq(rq); 590 queue->nr_cqe++; 591 592 return 0; 593 } 594 595 static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue, 596 struct nvme_tcp_data_pdu *pdu) 597 { 598 struct request *rq; 599 600 rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id); 601 if (!rq) { 602 dev_err(queue->ctrl->ctrl.device, 603 "got bad c2hdata.command_id %#x on queue %d\n", 604 pdu->command_id, nvme_tcp_queue_id(queue)); 605 return -ENOENT; 606 } 607 608 if (!blk_rq_payload_bytes(rq)) { 609 dev_err(queue->ctrl->ctrl.device, 610 "queue %d tag %#x unexpected data\n", 611 nvme_tcp_queue_id(queue), rq->tag); 612 return -EIO; 613 } 614 615 queue->data_remaining = le32_to_cpu(pdu->data_length); 616 617 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS && 618 unlikely(!(pdu->hdr.flags & NVME_TCP_F_DATA_LAST))) { 619 dev_err(queue->ctrl->ctrl.device, 620 "queue %d tag %#x SUCCESS set but not last PDU\n", 621 nvme_tcp_queue_id(queue), rq->tag); 622 nvme_tcp_error_recovery(&queue->ctrl->ctrl); 623 return -EPROTO; 624 } 625 626 return 0; 627 } 628 629 static int nvme_tcp_handle_comp(struct nvme_tcp_queue *queue, 630 struct nvme_tcp_rsp_pdu *pdu) 631 { 632 struct nvme_completion *cqe = &pdu->cqe; 633 int ret = 0; 634 635 /* 636 * AEN requests are special as they don't time out and can 637 * survive any kind of queue freeze and often don't respond to 638 * aborts. We don't even bother to allocate a struct request 639 * for them but rather special case them here. 640 */ 641 if (unlikely(nvme_is_aen_req(nvme_tcp_queue_id(queue), 642 cqe->command_id))) 643 nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status, 644 &cqe->result); 645 else 646 ret = nvme_tcp_process_nvme_cqe(queue, cqe); 647 648 return ret; 649 } 650 651 static void nvme_tcp_setup_h2c_data_pdu(struct nvme_tcp_request *req) 652 { 653 struct nvme_tcp_data_pdu *data = nvme_tcp_req_data_pdu(req); 654 struct nvme_tcp_queue *queue = req->queue; 655 struct request *rq = blk_mq_rq_from_pdu(req); 656 u32 h2cdata_sent = req->pdu_len; 657 u8 hdgst = nvme_tcp_hdgst_len(queue); 658 u8 ddgst = nvme_tcp_ddgst_len(queue); 659 660 req->state = NVME_TCP_SEND_H2C_PDU; 661 req->offset = 0; 662 req->pdu_len = min(req->h2cdata_left, queue->maxh2cdata); 663 req->pdu_sent = 0; 664 req->h2cdata_left -= req->pdu_len; 665 req->h2cdata_offset += h2cdata_sent; 666 667 memset(data, 0, sizeof(*data)); 668 data->hdr.type = nvme_tcp_h2c_data; 669 if (!req->h2cdata_left) 670 data->hdr.flags = NVME_TCP_F_DATA_LAST; 671 if (queue->hdr_digest) 672 data->hdr.flags |= NVME_TCP_F_HDGST; 673 if (queue->data_digest) 674 data->hdr.flags |= NVME_TCP_F_DDGST; 675 data->hdr.hlen = sizeof(*data); 676 data->hdr.pdo = data->hdr.hlen + hdgst; 677 data->hdr.plen = 678 cpu_to_le32(data->hdr.hlen + hdgst + req->pdu_len + ddgst); 679 data->ttag = req->ttag; 680 data->command_id = nvme_cid(rq); 681 data->data_offset = cpu_to_le32(req->h2cdata_offset); 682 data->data_length = cpu_to_le32(req->pdu_len); 683 } 684 685 static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue, 686 struct nvme_tcp_r2t_pdu *pdu) 687 { 688 struct nvme_tcp_request *req; 689 struct request *rq; 690 u32 r2t_length = le32_to_cpu(pdu->r2t_length); 691 u32 r2t_offset = le32_to_cpu(pdu->r2t_offset); 692 693 rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id); 694 if (!rq) { 695 dev_err(queue->ctrl->ctrl.device, 696 "got bad r2t.command_id %#x on queue %d\n", 697 pdu->command_id, nvme_tcp_queue_id(queue)); 698 return -ENOENT; 699 } 700 req = blk_mq_rq_to_pdu(rq); 701 702 if (unlikely(!r2t_length)) { 703 dev_err(queue->ctrl->ctrl.device, 704 "req %d r2t len is %u, probably a bug...\n", 705 rq->tag, r2t_length); 706 return -EPROTO; 707 } 708 709 if (unlikely(req->data_sent + r2t_length > req->data_len)) { 710 dev_err(queue->ctrl->ctrl.device, 711 "req %d r2t len %u exceeded data len %u (%zu sent)\n", 712 rq->tag, r2t_length, req->data_len, req->data_sent); 713 return -EPROTO; 714 } 715 716 if (unlikely(r2t_offset < req->data_sent)) { 717 dev_err(queue->ctrl->ctrl.device, 718 "req %d unexpected r2t offset %u (expected %zu)\n", 719 rq->tag, r2t_offset, req->data_sent); 720 return -EPROTO; 721 } 722 723 req->pdu_len = 0; 724 req->h2cdata_left = r2t_length; 725 req->h2cdata_offset = r2t_offset; 726 req->ttag = pdu->ttag; 727 728 nvme_tcp_setup_h2c_data_pdu(req); 729 nvme_tcp_queue_request(req, false, true); 730 731 return 0; 732 } 733 734 static int nvme_tcp_recv_pdu(struct nvme_tcp_queue *queue, struct sk_buff *skb, 735 unsigned int *offset, size_t *len) 736 { 737 struct nvme_tcp_hdr *hdr; 738 char *pdu = queue->pdu; 739 size_t rcv_len = min_t(size_t, *len, queue->pdu_remaining); 740 int ret; 741 742 ret = skb_copy_bits(skb, *offset, 743 &pdu[queue->pdu_offset], rcv_len); 744 if (unlikely(ret)) 745 return ret; 746 747 queue->pdu_remaining -= rcv_len; 748 queue->pdu_offset += rcv_len; 749 *offset += rcv_len; 750 *len -= rcv_len; 751 if (queue->pdu_remaining) 752 return 0; 753 754 hdr = queue->pdu; 755 if (queue->hdr_digest) { 756 ret = nvme_tcp_verify_hdgst(queue, queue->pdu, hdr->hlen); 757 if (unlikely(ret)) 758 return ret; 759 } 760 761 762 if (queue->data_digest) { 763 ret = nvme_tcp_check_ddgst(queue, queue->pdu); 764 if (unlikely(ret)) 765 return ret; 766 } 767 768 switch (hdr->type) { 769 case nvme_tcp_c2h_data: 770 return nvme_tcp_handle_c2h_data(queue, (void *)queue->pdu); 771 case nvme_tcp_rsp: 772 nvme_tcp_init_recv_ctx(queue); 773 return nvme_tcp_handle_comp(queue, (void *)queue->pdu); 774 case nvme_tcp_r2t: 775 nvme_tcp_init_recv_ctx(queue); 776 return nvme_tcp_handle_r2t(queue, (void *)queue->pdu); 777 default: 778 dev_err(queue->ctrl->ctrl.device, 779 "unsupported pdu type (%d)\n", hdr->type); 780 return -EINVAL; 781 } 782 } 783 784 static inline void nvme_tcp_end_request(struct request *rq, u16 status) 785 { 786 union nvme_result res = {}; 787 788 if (!nvme_try_complete_req(rq, cpu_to_le16(status << 1), res)) 789 nvme_complete_rq(rq); 790 } 791 792 static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb, 793 unsigned int *offset, size_t *len) 794 { 795 struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu; 796 struct request *rq = 797 nvme_cid_to_rq(nvme_tcp_tagset(queue), pdu->command_id); 798 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 799 800 while (true) { 801 int recv_len, ret; 802 803 recv_len = min_t(size_t, *len, queue->data_remaining); 804 if (!recv_len) 805 break; 806 807 if (!iov_iter_count(&req->iter)) { 808 req->curr_bio = req->curr_bio->bi_next; 809 810 /* 811 * If we don`t have any bios it means that controller 812 * sent more data than we requested, hence error 813 */ 814 if (!req->curr_bio) { 815 dev_err(queue->ctrl->ctrl.device, 816 "queue %d no space in request %#x", 817 nvme_tcp_queue_id(queue), rq->tag); 818 nvme_tcp_init_recv_ctx(queue); 819 return -EIO; 820 } 821 nvme_tcp_init_iter(req, ITER_DEST); 822 } 823 824 /* we can read only from what is left in this bio */ 825 recv_len = min_t(size_t, recv_len, 826 iov_iter_count(&req->iter)); 827 828 if (queue->data_digest) 829 ret = skb_copy_and_hash_datagram_iter(skb, *offset, 830 &req->iter, recv_len, queue->rcv_hash); 831 else 832 ret = skb_copy_datagram_iter(skb, *offset, 833 &req->iter, recv_len); 834 if (ret) { 835 dev_err(queue->ctrl->ctrl.device, 836 "queue %d failed to copy request %#x data", 837 nvme_tcp_queue_id(queue), rq->tag); 838 return ret; 839 } 840 841 *len -= recv_len; 842 *offset += recv_len; 843 queue->data_remaining -= recv_len; 844 } 845 846 if (!queue->data_remaining) { 847 if (queue->data_digest) { 848 nvme_tcp_ddgst_final(queue->rcv_hash, &queue->exp_ddgst); 849 queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH; 850 } else { 851 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) { 852 nvme_tcp_end_request(rq, 853 le16_to_cpu(req->status)); 854 queue->nr_cqe++; 855 } 856 nvme_tcp_init_recv_ctx(queue); 857 } 858 } 859 860 return 0; 861 } 862 863 static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue *queue, 864 struct sk_buff *skb, unsigned int *offset, size_t *len) 865 { 866 struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu; 867 char *ddgst = (char *)&queue->recv_ddgst; 868 size_t recv_len = min_t(size_t, *len, queue->ddgst_remaining); 869 off_t off = NVME_TCP_DIGEST_LENGTH - queue->ddgst_remaining; 870 int ret; 871 872 ret = skb_copy_bits(skb, *offset, &ddgst[off], recv_len); 873 if (unlikely(ret)) 874 return ret; 875 876 queue->ddgst_remaining -= recv_len; 877 *offset += recv_len; 878 *len -= recv_len; 879 if (queue->ddgst_remaining) 880 return 0; 881 882 if (queue->recv_ddgst != queue->exp_ddgst) { 883 struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue), 884 pdu->command_id); 885 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 886 887 req->status = cpu_to_le16(NVME_SC_DATA_XFER_ERROR); 888 889 dev_err(queue->ctrl->ctrl.device, 890 "data digest error: recv %#x expected %#x\n", 891 le32_to_cpu(queue->recv_ddgst), 892 le32_to_cpu(queue->exp_ddgst)); 893 } 894 895 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) { 896 struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue), 897 pdu->command_id); 898 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 899 900 nvme_tcp_end_request(rq, le16_to_cpu(req->status)); 901 queue->nr_cqe++; 902 } 903 904 nvme_tcp_init_recv_ctx(queue); 905 return 0; 906 } 907 908 static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb, 909 unsigned int offset, size_t len) 910 { 911 struct nvme_tcp_queue *queue = desc->arg.data; 912 size_t consumed = len; 913 int result; 914 915 if (unlikely(!queue->rd_enabled)) 916 return -EFAULT; 917 918 while (len) { 919 switch (nvme_tcp_recv_state(queue)) { 920 case NVME_TCP_RECV_PDU: 921 result = nvme_tcp_recv_pdu(queue, skb, &offset, &len); 922 break; 923 case NVME_TCP_RECV_DATA: 924 result = nvme_tcp_recv_data(queue, skb, &offset, &len); 925 break; 926 case NVME_TCP_RECV_DDGST: 927 result = nvme_tcp_recv_ddgst(queue, skb, &offset, &len); 928 break; 929 default: 930 result = -EFAULT; 931 } 932 if (result) { 933 dev_err(queue->ctrl->ctrl.device, 934 "receive failed: %d\n", result); 935 queue->rd_enabled = false; 936 nvme_tcp_error_recovery(&queue->ctrl->ctrl); 937 return result; 938 } 939 } 940 941 return consumed; 942 } 943 944 static void nvme_tcp_data_ready(struct sock *sk) 945 { 946 struct nvme_tcp_queue *queue; 947 948 trace_sk_data_ready(sk); 949 950 read_lock_bh(&sk->sk_callback_lock); 951 queue = sk->sk_user_data; 952 if (likely(queue && queue->rd_enabled) && 953 !test_bit(NVME_TCP_Q_POLLING, &queue->flags)) 954 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 955 read_unlock_bh(&sk->sk_callback_lock); 956 } 957 958 static void nvme_tcp_write_space(struct sock *sk) 959 { 960 struct nvme_tcp_queue *queue; 961 962 read_lock_bh(&sk->sk_callback_lock); 963 queue = sk->sk_user_data; 964 if (likely(queue && sk_stream_is_writeable(sk))) { 965 clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 966 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 967 } 968 read_unlock_bh(&sk->sk_callback_lock); 969 } 970 971 static void nvme_tcp_state_change(struct sock *sk) 972 { 973 struct nvme_tcp_queue *queue; 974 975 read_lock_bh(&sk->sk_callback_lock); 976 queue = sk->sk_user_data; 977 if (!queue) 978 goto done; 979 980 switch (sk->sk_state) { 981 case TCP_CLOSE: 982 case TCP_CLOSE_WAIT: 983 case TCP_LAST_ACK: 984 case TCP_FIN_WAIT1: 985 case TCP_FIN_WAIT2: 986 nvme_tcp_error_recovery(&queue->ctrl->ctrl); 987 break; 988 default: 989 dev_info(queue->ctrl->ctrl.device, 990 "queue %d socket state %d\n", 991 nvme_tcp_queue_id(queue), sk->sk_state); 992 } 993 994 queue->state_change(sk); 995 done: 996 read_unlock_bh(&sk->sk_callback_lock); 997 } 998 999 static inline void nvme_tcp_done_send_req(struct nvme_tcp_queue *queue) 1000 { 1001 queue->request = NULL; 1002 } 1003 1004 static void nvme_tcp_fail_request(struct nvme_tcp_request *req) 1005 { 1006 if (nvme_tcp_async_req(req)) { 1007 union nvme_result res = {}; 1008 1009 nvme_complete_async_event(&req->queue->ctrl->ctrl, 1010 cpu_to_le16(NVME_SC_HOST_PATH_ERROR), &res); 1011 } else { 1012 nvme_tcp_end_request(blk_mq_rq_from_pdu(req), 1013 NVME_SC_HOST_PATH_ERROR); 1014 } 1015 } 1016 1017 static int nvme_tcp_try_send_data(struct nvme_tcp_request *req) 1018 { 1019 struct nvme_tcp_queue *queue = req->queue; 1020 int req_data_len = req->data_len; 1021 u32 h2cdata_left = req->h2cdata_left; 1022 1023 while (true) { 1024 struct bio_vec bvec; 1025 struct msghdr msg = { 1026 .msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES, 1027 }; 1028 struct page *page = nvme_tcp_req_cur_page(req); 1029 size_t offset = nvme_tcp_req_cur_offset(req); 1030 size_t len = nvme_tcp_req_cur_length(req); 1031 bool last = nvme_tcp_pdu_last_send(req, len); 1032 int req_data_sent = req->data_sent; 1033 int ret; 1034 1035 if (last && !queue->data_digest && !nvme_tcp_queue_more(queue)) 1036 msg.msg_flags |= MSG_EOR; 1037 else 1038 msg.msg_flags |= MSG_MORE; 1039 1040 if (!sendpage_ok(page)) 1041 msg.msg_flags &= ~MSG_SPLICE_PAGES; 1042 1043 bvec_set_page(&bvec, page, len, offset); 1044 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, len); 1045 ret = sock_sendmsg(queue->sock, &msg); 1046 if (ret <= 0) 1047 return ret; 1048 1049 if (queue->data_digest) 1050 nvme_tcp_ddgst_update(queue->snd_hash, page, 1051 offset, ret); 1052 1053 /* 1054 * update the request iterator except for the last payload send 1055 * in the request where we don't want to modify it as we may 1056 * compete with the RX path completing the request. 1057 */ 1058 if (req_data_sent + ret < req_data_len) 1059 nvme_tcp_advance_req(req, ret); 1060 1061 /* fully successful last send in current PDU */ 1062 if (last && ret == len) { 1063 if (queue->data_digest) { 1064 nvme_tcp_ddgst_final(queue->snd_hash, 1065 &req->ddgst); 1066 req->state = NVME_TCP_SEND_DDGST; 1067 req->offset = 0; 1068 } else { 1069 if (h2cdata_left) 1070 nvme_tcp_setup_h2c_data_pdu(req); 1071 else 1072 nvme_tcp_done_send_req(queue); 1073 } 1074 return 1; 1075 } 1076 } 1077 return -EAGAIN; 1078 } 1079 1080 static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req) 1081 { 1082 struct nvme_tcp_queue *queue = req->queue; 1083 struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req); 1084 struct bio_vec bvec; 1085 struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES, }; 1086 bool inline_data = nvme_tcp_has_inline_data(req); 1087 u8 hdgst = nvme_tcp_hdgst_len(queue); 1088 int len = sizeof(*pdu) + hdgst - req->offset; 1089 int ret; 1090 1091 if (inline_data || nvme_tcp_queue_more(queue)) 1092 msg.msg_flags |= MSG_MORE; 1093 else 1094 msg.msg_flags |= MSG_EOR; 1095 1096 if (queue->hdr_digest && !req->offset) 1097 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu)); 1098 1099 bvec_set_virt(&bvec, (void *)pdu + req->offset, len); 1100 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, len); 1101 ret = sock_sendmsg(queue->sock, &msg); 1102 if (unlikely(ret <= 0)) 1103 return ret; 1104 1105 len -= ret; 1106 if (!len) { 1107 if (inline_data) { 1108 req->state = NVME_TCP_SEND_DATA; 1109 if (queue->data_digest) 1110 crypto_ahash_init(queue->snd_hash); 1111 } else { 1112 nvme_tcp_done_send_req(queue); 1113 } 1114 return 1; 1115 } 1116 req->offset += ret; 1117 1118 return -EAGAIN; 1119 } 1120 1121 static int nvme_tcp_try_send_data_pdu(struct nvme_tcp_request *req) 1122 { 1123 struct nvme_tcp_queue *queue = req->queue; 1124 struct nvme_tcp_data_pdu *pdu = nvme_tcp_req_data_pdu(req); 1125 struct bio_vec bvec; 1126 struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_MORE, }; 1127 u8 hdgst = nvme_tcp_hdgst_len(queue); 1128 int len = sizeof(*pdu) - req->offset + hdgst; 1129 int ret; 1130 1131 if (queue->hdr_digest && !req->offset) 1132 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu)); 1133 1134 if (!req->h2cdata_left) 1135 msg.msg_flags |= MSG_SPLICE_PAGES; 1136 1137 bvec_set_virt(&bvec, (void *)pdu + req->offset, len); 1138 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, len); 1139 ret = sock_sendmsg(queue->sock, &msg); 1140 if (unlikely(ret <= 0)) 1141 return ret; 1142 1143 len -= ret; 1144 if (!len) { 1145 req->state = NVME_TCP_SEND_DATA; 1146 if (queue->data_digest) 1147 crypto_ahash_init(queue->snd_hash); 1148 return 1; 1149 } 1150 req->offset += ret; 1151 1152 return -EAGAIN; 1153 } 1154 1155 static int nvme_tcp_try_send_ddgst(struct nvme_tcp_request *req) 1156 { 1157 struct nvme_tcp_queue *queue = req->queue; 1158 size_t offset = req->offset; 1159 u32 h2cdata_left = req->h2cdata_left; 1160 int ret; 1161 struct msghdr msg = { .msg_flags = MSG_DONTWAIT }; 1162 struct kvec iov = { 1163 .iov_base = (u8 *)&req->ddgst + req->offset, 1164 .iov_len = NVME_TCP_DIGEST_LENGTH - req->offset 1165 }; 1166 1167 if (nvme_tcp_queue_more(queue)) 1168 msg.msg_flags |= MSG_MORE; 1169 else 1170 msg.msg_flags |= MSG_EOR; 1171 1172 ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len); 1173 if (unlikely(ret <= 0)) 1174 return ret; 1175 1176 if (offset + ret == NVME_TCP_DIGEST_LENGTH) { 1177 if (h2cdata_left) 1178 nvme_tcp_setup_h2c_data_pdu(req); 1179 else 1180 nvme_tcp_done_send_req(queue); 1181 return 1; 1182 } 1183 1184 req->offset += ret; 1185 return -EAGAIN; 1186 } 1187 1188 static int nvme_tcp_try_send(struct nvme_tcp_queue *queue) 1189 { 1190 struct nvme_tcp_request *req; 1191 unsigned int noreclaim_flag; 1192 int ret = 1; 1193 1194 if (!queue->request) { 1195 queue->request = nvme_tcp_fetch_request(queue); 1196 if (!queue->request) 1197 return 0; 1198 } 1199 req = queue->request; 1200 1201 noreclaim_flag = memalloc_noreclaim_save(); 1202 if (req->state == NVME_TCP_SEND_CMD_PDU) { 1203 ret = nvme_tcp_try_send_cmd_pdu(req); 1204 if (ret <= 0) 1205 goto done; 1206 if (!nvme_tcp_has_inline_data(req)) 1207 goto out; 1208 } 1209 1210 if (req->state == NVME_TCP_SEND_H2C_PDU) { 1211 ret = nvme_tcp_try_send_data_pdu(req); 1212 if (ret <= 0) 1213 goto done; 1214 } 1215 1216 if (req->state == NVME_TCP_SEND_DATA) { 1217 ret = nvme_tcp_try_send_data(req); 1218 if (ret <= 0) 1219 goto done; 1220 } 1221 1222 if (req->state == NVME_TCP_SEND_DDGST) 1223 ret = nvme_tcp_try_send_ddgst(req); 1224 done: 1225 if (ret == -EAGAIN) { 1226 ret = 0; 1227 } else if (ret < 0) { 1228 dev_err(queue->ctrl->ctrl.device, 1229 "failed to send request %d\n", ret); 1230 nvme_tcp_fail_request(queue->request); 1231 nvme_tcp_done_send_req(queue); 1232 } 1233 out: 1234 memalloc_noreclaim_restore(noreclaim_flag); 1235 return ret; 1236 } 1237 1238 static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue) 1239 { 1240 struct socket *sock = queue->sock; 1241 struct sock *sk = sock->sk; 1242 read_descriptor_t rd_desc; 1243 int consumed; 1244 1245 rd_desc.arg.data = queue; 1246 rd_desc.count = 1; 1247 lock_sock(sk); 1248 queue->nr_cqe = 0; 1249 consumed = sock->ops->read_sock(sk, &rd_desc, nvme_tcp_recv_skb); 1250 release_sock(sk); 1251 return consumed; 1252 } 1253 1254 static void nvme_tcp_io_work(struct work_struct *w) 1255 { 1256 struct nvme_tcp_queue *queue = 1257 container_of(w, struct nvme_tcp_queue, io_work); 1258 unsigned long deadline = jiffies + msecs_to_jiffies(1); 1259 1260 do { 1261 bool pending = false; 1262 int result; 1263 1264 if (mutex_trylock(&queue->send_mutex)) { 1265 result = nvme_tcp_try_send(queue); 1266 mutex_unlock(&queue->send_mutex); 1267 if (result > 0) 1268 pending = true; 1269 else if (unlikely(result < 0)) 1270 break; 1271 } 1272 1273 result = nvme_tcp_try_recv(queue); 1274 if (result > 0) 1275 pending = true; 1276 else if (unlikely(result < 0)) 1277 return; 1278 1279 if (!pending || !queue->rd_enabled) 1280 return; 1281 1282 } while (!time_after(jiffies, deadline)); /* quota is exhausted */ 1283 1284 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 1285 } 1286 1287 static void nvme_tcp_free_crypto(struct nvme_tcp_queue *queue) 1288 { 1289 struct crypto_ahash *tfm = crypto_ahash_reqtfm(queue->rcv_hash); 1290 1291 ahash_request_free(queue->rcv_hash); 1292 ahash_request_free(queue->snd_hash); 1293 crypto_free_ahash(tfm); 1294 } 1295 1296 static int nvme_tcp_alloc_crypto(struct nvme_tcp_queue *queue) 1297 { 1298 struct crypto_ahash *tfm; 1299 1300 tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC); 1301 if (IS_ERR(tfm)) 1302 return PTR_ERR(tfm); 1303 1304 queue->snd_hash = ahash_request_alloc(tfm, GFP_KERNEL); 1305 if (!queue->snd_hash) 1306 goto free_tfm; 1307 ahash_request_set_callback(queue->snd_hash, 0, NULL, NULL); 1308 1309 queue->rcv_hash = ahash_request_alloc(tfm, GFP_KERNEL); 1310 if (!queue->rcv_hash) 1311 goto free_snd_hash; 1312 ahash_request_set_callback(queue->rcv_hash, 0, NULL, NULL); 1313 1314 return 0; 1315 free_snd_hash: 1316 ahash_request_free(queue->snd_hash); 1317 free_tfm: 1318 crypto_free_ahash(tfm); 1319 return -ENOMEM; 1320 } 1321 1322 static void nvme_tcp_free_async_req(struct nvme_tcp_ctrl *ctrl) 1323 { 1324 struct nvme_tcp_request *async = &ctrl->async_req; 1325 1326 page_frag_free(async->pdu); 1327 } 1328 1329 static int nvme_tcp_alloc_async_req(struct nvme_tcp_ctrl *ctrl) 1330 { 1331 struct nvme_tcp_queue *queue = &ctrl->queues[0]; 1332 struct nvme_tcp_request *async = &ctrl->async_req; 1333 u8 hdgst = nvme_tcp_hdgst_len(queue); 1334 1335 async->pdu = page_frag_alloc(&queue->pf_cache, 1336 sizeof(struct nvme_tcp_cmd_pdu) + hdgst, 1337 GFP_KERNEL | __GFP_ZERO); 1338 if (!async->pdu) 1339 return -ENOMEM; 1340 1341 async->queue = &ctrl->queues[0]; 1342 return 0; 1343 } 1344 1345 static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid) 1346 { 1347 struct page *page; 1348 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1349 struct nvme_tcp_queue *queue = &ctrl->queues[qid]; 1350 unsigned int noreclaim_flag; 1351 1352 if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags)) 1353 return; 1354 1355 if (queue->hdr_digest || queue->data_digest) 1356 nvme_tcp_free_crypto(queue); 1357 1358 if (queue->pf_cache.va) { 1359 page = virt_to_head_page(queue->pf_cache.va); 1360 __page_frag_cache_drain(page, queue->pf_cache.pagecnt_bias); 1361 queue->pf_cache.va = NULL; 1362 } 1363 1364 noreclaim_flag = memalloc_noreclaim_save(); 1365 /* ->sock will be released by fput() */ 1366 fput(queue->sock->file); 1367 queue->sock = NULL; 1368 memalloc_noreclaim_restore(noreclaim_flag); 1369 1370 kfree(queue->pdu); 1371 mutex_destroy(&queue->send_mutex); 1372 mutex_destroy(&queue->queue_lock); 1373 } 1374 1375 static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue) 1376 { 1377 struct nvme_tcp_icreq_pdu *icreq; 1378 struct nvme_tcp_icresp_pdu *icresp; 1379 char cbuf[CMSG_LEN(sizeof(char))] = {}; 1380 u8 ctype; 1381 struct msghdr msg = {}; 1382 struct kvec iov; 1383 bool ctrl_hdgst, ctrl_ddgst; 1384 u32 maxh2cdata; 1385 int ret; 1386 1387 icreq = kzalloc(sizeof(*icreq), GFP_KERNEL); 1388 if (!icreq) 1389 return -ENOMEM; 1390 1391 icresp = kzalloc(sizeof(*icresp), GFP_KERNEL); 1392 if (!icresp) { 1393 ret = -ENOMEM; 1394 goto free_icreq; 1395 } 1396 1397 icreq->hdr.type = nvme_tcp_icreq; 1398 icreq->hdr.hlen = sizeof(*icreq); 1399 icreq->hdr.pdo = 0; 1400 icreq->hdr.plen = cpu_to_le32(icreq->hdr.hlen); 1401 icreq->pfv = cpu_to_le16(NVME_TCP_PFV_1_0); 1402 icreq->maxr2t = 0; /* single inflight r2t supported */ 1403 icreq->hpda = 0; /* no alignment constraint */ 1404 if (queue->hdr_digest) 1405 icreq->digest |= NVME_TCP_HDR_DIGEST_ENABLE; 1406 if (queue->data_digest) 1407 icreq->digest |= NVME_TCP_DATA_DIGEST_ENABLE; 1408 1409 iov.iov_base = icreq; 1410 iov.iov_len = sizeof(*icreq); 1411 ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len); 1412 if (ret < 0) { 1413 pr_warn("queue %d: failed to send icreq, error %d\n", 1414 nvme_tcp_queue_id(queue), ret); 1415 goto free_icresp; 1416 } 1417 1418 memset(&msg, 0, sizeof(msg)); 1419 iov.iov_base = icresp; 1420 iov.iov_len = sizeof(*icresp); 1421 if (nvme_tcp_tls(&queue->ctrl->ctrl)) { 1422 msg.msg_control = cbuf; 1423 msg.msg_controllen = sizeof(cbuf); 1424 } 1425 ret = kernel_recvmsg(queue->sock, &msg, &iov, 1, 1426 iov.iov_len, msg.msg_flags); 1427 if (ret < 0) { 1428 pr_warn("queue %d: failed to receive icresp, error %d\n", 1429 nvme_tcp_queue_id(queue), ret); 1430 goto free_icresp; 1431 } 1432 ret = -ENOTCONN; 1433 if (nvme_tcp_tls(&queue->ctrl->ctrl)) { 1434 ctype = tls_get_record_type(queue->sock->sk, 1435 (struct cmsghdr *)cbuf); 1436 if (ctype != TLS_RECORD_TYPE_DATA) { 1437 pr_err("queue %d: unhandled TLS record %d\n", 1438 nvme_tcp_queue_id(queue), ctype); 1439 goto free_icresp; 1440 } 1441 } 1442 ret = -EINVAL; 1443 if (icresp->hdr.type != nvme_tcp_icresp) { 1444 pr_err("queue %d: bad type returned %d\n", 1445 nvme_tcp_queue_id(queue), icresp->hdr.type); 1446 goto free_icresp; 1447 } 1448 1449 if (le32_to_cpu(icresp->hdr.plen) != sizeof(*icresp)) { 1450 pr_err("queue %d: bad pdu length returned %d\n", 1451 nvme_tcp_queue_id(queue), icresp->hdr.plen); 1452 goto free_icresp; 1453 } 1454 1455 if (icresp->pfv != NVME_TCP_PFV_1_0) { 1456 pr_err("queue %d: bad pfv returned %d\n", 1457 nvme_tcp_queue_id(queue), icresp->pfv); 1458 goto free_icresp; 1459 } 1460 1461 ctrl_ddgst = !!(icresp->digest & NVME_TCP_DATA_DIGEST_ENABLE); 1462 if ((queue->data_digest && !ctrl_ddgst) || 1463 (!queue->data_digest && ctrl_ddgst)) { 1464 pr_err("queue %d: data digest mismatch host: %s ctrl: %s\n", 1465 nvme_tcp_queue_id(queue), 1466 queue->data_digest ? "enabled" : "disabled", 1467 ctrl_ddgst ? "enabled" : "disabled"); 1468 goto free_icresp; 1469 } 1470 1471 ctrl_hdgst = !!(icresp->digest & NVME_TCP_HDR_DIGEST_ENABLE); 1472 if ((queue->hdr_digest && !ctrl_hdgst) || 1473 (!queue->hdr_digest && ctrl_hdgst)) { 1474 pr_err("queue %d: header digest mismatch host: %s ctrl: %s\n", 1475 nvme_tcp_queue_id(queue), 1476 queue->hdr_digest ? "enabled" : "disabled", 1477 ctrl_hdgst ? "enabled" : "disabled"); 1478 goto free_icresp; 1479 } 1480 1481 if (icresp->cpda != 0) { 1482 pr_err("queue %d: unsupported cpda returned %d\n", 1483 nvme_tcp_queue_id(queue), icresp->cpda); 1484 goto free_icresp; 1485 } 1486 1487 maxh2cdata = le32_to_cpu(icresp->maxdata); 1488 if ((maxh2cdata % 4) || (maxh2cdata < NVME_TCP_MIN_MAXH2CDATA)) { 1489 pr_err("queue %d: invalid maxh2cdata returned %u\n", 1490 nvme_tcp_queue_id(queue), maxh2cdata); 1491 goto free_icresp; 1492 } 1493 queue->maxh2cdata = maxh2cdata; 1494 1495 ret = 0; 1496 free_icresp: 1497 kfree(icresp); 1498 free_icreq: 1499 kfree(icreq); 1500 return ret; 1501 } 1502 1503 static bool nvme_tcp_admin_queue(struct nvme_tcp_queue *queue) 1504 { 1505 return nvme_tcp_queue_id(queue) == 0; 1506 } 1507 1508 static bool nvme_tcp_default_queue(struct nvme_tcp_queue *queue) 1509 { 1510 struct nvme_tcp_ctrl *ctrl = queue->ctrl; 1511 int qid = nvme_tcp_queue_id(queue); 1512 1513 return !nvme_tcp_admin_queue(queue) && 1514 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT]; 1515 } 1516 1517 static bool nvme_tcp_read_queue(struct nvme_tcp_queue *queue) 1518 { 1519 struct nvme_tcp_ctrl *ctrl = queue->ctrl; 1520 int qid = nvme_tcp_queue_id(queue); 1521 1522 return !nvme_tcp_admin_queue(queue) && 1523 !nvme_tcp_default_queue(queue) && 1524 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] + 1525 ctrl->io_queues[HCTX_TYPE_READ]; 1526 } 1527 1528 static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue) 1529 { 1530 struct nvme_tcp_ctrl *ctrl = queue->ctrl; 1531 int qid = nvme_tcp_queue_id(queue); 1532 1533 return !nvme_tcp_admin_queue(queue) && 1534 !nvme_tcp_default_queue(queue) && 1535 !nvme_tcp_read_queue(queue) && 1536 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] + 1537 ctrl->io_queues[HCTX_TYPE_READ] + 1538 ctrl->io_queues[HCTX_TYPE_POLL]; 1539 } 1540 1541 static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue) 1542 { 1543 struct nvme_tcp_ctrl *ctrl = queue->ctrl; 1544 int qid = nvme_tcp_queue_id(queue); 1545 int n = 0; 1546 1547 if (nvme_tcp_default_queue(queue)) 1548 n = qid - 1; 1549 else if (nvme_tcp_read_queue(queue)) 1550 n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] - 1; 1551 else if (nvme_tcp_poll_queue(queue)) 1552 n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] - 1553 ctrl->io_queues[HCTX_TYPE_READ] - 1; 1554 queue->io_cpu = cpumask_next_wrap(n - 1, cpu_online_mask, -1, false); 1555 } 1556 1557 static void nvme_tcp_tls_done(void *data, int status, key_serial_t pskid) 1558 { 1559 struct nvme_tcp_queue *queue = data; 1560 struct nvme_tcp_ctrl *ctrl = queue->ctrl; 1561 int qid = nvme_tcp_queue_id(queue); 1562 struct key *tls_key; 1563 1564 dev_dbg(ctrl->ctrl.device, "queue %d: TLS handshake done, key %x, status %d\n", 1565 qid, pskid, status); 1566 1567 if (status) { 1568 queue->tls_err = -status; 1569 goto out_complete; 1570 } 1571 1572 tls_key = key_lookup(pskid); 1573 if (IS_ERR(tls_key)) { 1574 dev_warn(ctrl->ctrl.device, "queue %d: Invalid key %x\n", 1575 qid, pskid); 1576 queue->tls_err = -ENOKEY; 1577 } else { 1578 ctrl->ctrl.tls_key = tls_key; 1579 queue->tls_err = 0; 1580 } 1581 1582 out_complete: 1583 complete(&queue->tls_complete); 1584 } 1585 1586 static int nvme_tcp_start_tls(struct nvme_ctrl *nctrl, 1587 struct nvme_tcp_queue *queue, 1588 key_serial_t pskid) 1589 { 1590 int qid = nvme_tcp_queue_id(queue); 1591 int ret; 1592 struct tls_handshake_args args; 1593 unsigned long tmo = tls_handshake_timeout * HZ; 1594 key_serial_t keyring = nvme_keyring_id(); 1595 1596 dev_dbg(nctrl->device, "queue %d: start TLS with key %x\n", 1597 qid, pskid); 1598 memset(&args, 0, sizeof(args)); 1599 args.ta_sock = queue->sock; 1600 args.ta_done = nvme_tcp_tls_done; 1601 args.ta_data = queue; 1602 args.ta_my_peerids[0] = pskid; 1603 args.ta_num_peerids = 1; 1604 if (nctrl->opts->keyring) 1605 keyring = key_serial(nctrl->opts->keyring); 1606 args.ta_keyring = keyring; 1607 args.ta_timeout_ms = tls_handshake_timeout * 1000; 1608 queue->tls_err = -EOPNOTSUPP; 1609 init_completion(&queue->tls_complete); 1610 ret = tls_client_hello_psk(&args, GFP_KERNEL); 1611 if (ret) { 1612 dev_err(nctrl->device, "queue %d: failed to start TLS: %d\n", 1613 qid, ret); 1614 return ret; 1615 } 1616 ret = wait_for_completion_interruptible_timeout(&queue->tls_complete, tmo); 1617 if (ret <= 0) { 1618 if (ret == 0) 1619 ret = -ETIMEDOUT; 1620 1621 dev_err(nctrl->device, 1622 "queue %d: TLS handshake failed, error %d\n", 1623 qid, ret); 1624 tls_handshake_cancel(queue->sock->sk); 1625 } else { 1626 dev_dbg(nctrl->device, 1627 "queue %d: TLS handshake complete, error %d\n", 1628 qid, queue->tls_err); 1629 ret = queue->tls_err; 1630 } 1631 return ret; 1632 } 1633 1634 static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid, 1635 key_serial_t pskid) 1636 { 1637 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1638 struct nvme_tcp_queue *queue = &ctrl->queues[qid]; 1639 int ret, rcv_pdu_size; 1640 struct file *sock_file; 1641 1642 mutex_init(&queue->queue_lock); 1643 queue->ctrl = ctrl; 1644 init_llist_head(&queue->req_list); 1645 INIT_LIST_HEAD(&queue->send_list); 1646 mutex_init(&queue->send_mutex); 1647 INIT_WORK(&queue->io_work, nvme_tcp_io_work); 1648 1649 if (qid > 0) 1650 queue->cmnd_capsule_len = nctrl->ioccsz * 16; 1651 else 1652 queue->cmnd_capsule_len = sizeof(struct nvme_command) + 1653 NVME_TCP_ADMIN_CCSZ; 1654 1655 ret = sock_create(ctrl->addr.ss_family, SOCK_STREAM, 1656 IPPROTO_TCP, &queue->sock); 1657 if (ret) { 1658 dev_err(nctrl->device, 1659 "failed to create socket: %d\n", ret); 1660 goto err_destroy_mutex; 1661 } 1662 1663 sock_file = sock_alloc_file(queue->sock, O_CLOEXEC, NULL); 1664 if (IS_ERR(sock_file)) { 1665 ret = PTR_ERR(sock_file); 1666 goto err_destroy_mutex; 1667 } 1668 nvme_tcp_reclassify_socket(queue->sock); 1669 1670 /* Single syn retry */ 1671 tcp_sock_set_syncnt(queue->sock->sk, 1); 1672 1673 /* Set TCP no delay */ 1674 tcp_sock_set_nodelay(queue->sock->sk); 1675 1676 /* 1677 * Cleanup whatever is sitting in the TCP transmit queue on socket 1678 * close. This is done to prevent stale data from being sent should 1679 * the network connection be restored before TCP times out. 1680 */ 1681 sock_no_linger(queue->sock->sk); 1682 1683 if (so_priority > 0) 1684 sock_set_priority(queue->sock->sk, so_priority); 1685 1686 /* Set socket type of service */ 1687 if (nctrl->opts->tos >= 0) 1688 ip_sock_set_tos(queue->sock->sk, nctrl->opts->tos); 1689 1690 /* Set 10 seconds timeout for icresp recvmsg */ 1691 queue->sock->sk->sk_rcvtimeo = 10 * HZ; 1692 1693 queue->sock->sk->sk_allocation = GFP_ATOMIC; 1694 queue->sock->sk->sk_use_task_frag = false; 1695 nvme_tcp_set_queue_io_cpu(queue); 1696 queue->request = NULL; 1697 queue->data_remaining = 0; 1698 queue->ddgst_remaining = 0; 1699 queue->pdu_remaining = 0; 1700 queue->pdu_offset = 0; 1701 sk_set_memalloc(queue->sock->sk); 1702 1703 if (nctrl->opts->mask & NVMF_OPT_HOST_TRADDR) { 1704 ret = kernel_bind(queue->sock, (struct sockaddr *)&ctrl->src_addr, 1705 sizeof(ctrl->src_addr)); 1706 if (ret) { 1707 dev_err(nctrl->device, 1708 "failed to bind queue %d socket %d\n", 1709 qid, ret); 1710 goto err_sock; 1711 } 1712 } 1713 1714 if (nctrl->opts->mask & NVMF_OPT_HOST_IFACE) { 1715 char *iface = nctrl->opts->host_iface; 1716 sockptr_t optval = KERNEL_SOCKPTR(iface); 1717 1718 ret = sock_setsockopt(queue->sock, SOL_SOCKET, SO_BINDTODEVICE, 1719 optval, strlen(iface)); 1720 if (ret) { 1721 dev_err(nctrl->device, 1722 "failed to bind to interface %s queue %d err %d\n", 1723 iface, qid, ret); 1724 goto err_sock; 1725 } 1726 } 1727 1728 queue->hdr_digest = nctrl->opts->hdr_digest; 1729 queue->data_digest = nctrl->opts->data_digest; 1730 if (queue->hdr_digest || queue->data_digest) { 1731 ret = nvme_tcp_alloc_crypto(queue); 1732 if (ret) { 1733 dev_err(nctrl->device, 1734 "failed to allocate queue %d crypto\n", qid); 1735 goto err_sock; 1736 } 1737 } 1738 1739 rcv_pdu_size = sizeof(struct nvme_tcp_rsp_pdu) + 1740 nvme_tcp_hdgst_len(queue); 1741 queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL); 1742 if (!queue->pdu) { 1743 ret = -ENOMEM; 1744 goto err_crypto; 1745 } 1746 1747 dev_dbg(nctrl->device, "connecting queue %d\n", 1748 nvme_tcp_queue_id(queue)); 1749 1750 ret = kernel_connect(queue->sock, (struct sockaddr *)&ctrl->addr, 1751 sizeof(ctrl->addr), 0); 1752 if (ret) { 1753 dev_err(nctrl->device, 1754 "failed to connect socket: %d\n", ret); 1755 goto err_rcv_pdu; 1756 } 1757 1758 /* If PSKs are configured try to start TLS */ 1759 if (IS_ENABLED(CONFIG_NVME_TCP_TLS) && pskid) { 1760 ret = nvme_tcp_start_tls(nctrl, queue, pskid); 1761 if (ret) 1762 goto err_init_connect; 1763 } 1764 1765 ret = nvme_tcp_init_connection(queue); 1766 if (ret) 1767 goto err_init_connect; 1768 1769 set_bit(NVME_TCP_Q_ALLOCATED, &queue->flags); 1770 1771 return 0; 1772 1773 err_init_connect: 1774 kernel_sock_shutdown(queue->sock, SHUT_RDWR); 1775 err_rcv_pdu: 1776 kfree(queue->pdu); 1777 err_crypto: 1778 if (queue->hdr_digest || queue->data_digest) 1779 nvme_tcp_free_crypto(queue); 1780 err_sock: 1781 /* ->sock will be released by fput() */ 1782 fput(queue->sock->file); 1783 queue->sock = NULL; 1784 err_destroy_mutex: 1785 mutex_destroy(&queue->send_mutex); 1786 mutex_destroy(&queue->queue_lock); 1787 return ret; 1788 } 1789 1790 static void nvme_tcp_restore_sock_ops(struct nvme_tcp_queue *queue) 1791 { 1792 struct socket *sock = queue->sock; 1793 1794 write_lock_bh(&sock->sk->sk_callback_lock); 1795 sock->sk->sk_user_data = NULL; 1796 sock->sk->sk_data_ready = queue->data_ready; 1797 sock->sk->sk_state_change = queue->state_change; 1798 sock->sk->sk_write_space = queue->write_space; 1799 write_unlock_bh(&sock->sk->sk_callback_lock); 1800 } 1801 1802 static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue) 1803 { 1804 kernel_sock_shutdown(queue->sock, SHUT_RDWR); 1805 nvme_tcp_restore_sock_ops(queue); 1806 cancel_work_sync(&queue->io_work); 1807 } 1808 1809 static void nvme_tcp_stop_queue(struct nvme_ctrl *nctrl, int qid) 1810 { 1811 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1812 struct nvme_tcp_queue *queue = &ctrl->queues[qid]; 1813 1814 if (!test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags)) 1815 return; 1816 1817 mutex_lock(&queue->queue_lock); 1818 if (test_and_clear_bit(NVME_TCP_Q_LIVE, &queue->flags)) 1819 __nvme_tcp_stop_queue(queue); 1820 mutex_unlock(&queue->queue_lock); 1821 } 1822 1823 static void nvme_tcp_setup_sock_ops(struct nvme_tcp_queue *queue) 1824 { 1825 write_lock_bh(&queue->sock->sk->sk_callback_lock); 1826 queue->sock->sk->sk_user_data = queue; 1827 queue->state_change = queue->sock->sk->sk_state_change; 1828 queue->data_ready = queue->sock->sk->sk_data_ready; 1829 queue->write_space = queue->sock->sk->sk_write_space; 1830 queue->sock->sk->sk_data_ready = nvme_tcp_data_ready; 1831 queue->sock->sk->sk_state_change = nvme_tcp_state_change; 1832 queue->sock->sk->sk_write_space = nvme_tcp_write_space; 1833 #ifdef CONFIG_NET_RX_BUSY_POLL 1834 queue->sock->sk->sk_ll_usec = 1; 1835 #endif 1836 write_unlock_bh(&queue->sock->sk->sk_callback_lock); 1837 } 1838 1839 static int nvme_tcp_start_queue(struct nvme_ctrl *nctrl, int idx) 1840 { 1841 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1842 struct nvme_tcp_queue *queue = &ctrl->queues[idx]; 1843 int ret; 1844 1845 queue->rd_enabled = true; 1846 nvme_tcp_init_recv_ctx(queue); 1847 nvme_tcp_setup_sock_ops(queue); 1848 1849 if (idx) 1850 ret = nvmf_connect_io_queue(nctrl, idx); 1851 else 1852 ret = nvmf_connect_admin_queue(nctrl); 1853 1854 if (!ret) { 1855 set_bit(NVME_TCP_Q_LIVE, &queue->flags); 1856 } else { 1857 if (test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags)) 1858 __nvme_tcp_stop_queue(queue); 1859 dev_err(nctrl->device, 1860 "failed to connect queue: %d ret=%d\n", idx, ret); 1861 } 1862 return ret; 1863 } 1864 1865 static void nvme_tcp_free_admin_queue(struct nvme_ctrl *ctrl) 1866 { 1867 if (to_tcp_ctrl(ctrl)->async_req.pdu) { 1868 cancel_work_sync(&ctrl->async_event_work); 1869 nvme_tcp_free_async_req(to_tcp_ctrl(ctrl)); 1870 to_tcp_ctrl(ctrl)->async_req.pdu = NULL; 1871 } 1872 1873 nvme_tcp_free_queue(ctrl, 0); 1874 } 1875 1876 static void nvme_tcp_free_io_queues(struct nvme_ctrl *ctrl) 1877 { 1878 int i; 1879 1880 for (i = 1; i < ctrl->queue_count; i++) 1881 nvme_tcp_free_queue(ctrl, i); 1882 } 1883 1884 static void nvme_tcp_stop_io_queues(struct nvme_ctrl *ctrl) 1885 { 1886 int i; 1887 1888 for (i = 1; i < ctrl->queue_count; i++) 1889 nvme_tcp_stop_queue(ctrl, i); 1890 } 1891 1892 static int nvme_tcp_start_io_queues(struct nvme_ctrl *ctrl, 1893 int first, int last) 1894 { 1895 int i, ret; 1896 1897 for (i = first; i < last; i++) { 1898 ret = nvme_tcp_start_queue(ctrl, i); 1899 if (ret) 1900 goto out_stop_queues; 1901 } 1902 1903 return 0; 1904 1905 out_stop_queues: 1906 for (i--; i >= first; i--) 1907 nvme_tcp_stop_queue(ctrl, i); 1908 return ret; 1909 } 1910 1911 static int nvme_tcp_alloc_admin_queue(struct nvme_ctrl *ctrl) 1912 { 1913 int ret; 1914 key_serial_t pskid = 0; 1915 1916 if (nvme_tcp_tls(ctrl)) { 1917 if (ctrl->opts->tls_key) 1918 pskid = key_serial(ctrl->opts->tls_key); 1919 else 1920 pskid = nvme_tls_psk_default(ctrl->opts->keyring, 1921 ctrl->opts->host->nqn, 1922 ctrl->opts->subsysnqn); 1923 if (!pskid) { 1924 dev_err(ctrl->device, "no valid PSK found\n"); 1925 ret = -ENOKEY; 1926 goto out_free_queue; 1927 } 1928 } 1929 1930 ret = nvme_tcp_alloc_queue(ctrl, 0, pskid); 1931 if (ret) 1932 goto out_free_queue; 1933 1934 ret = nvme_tcp_alloc_async_req(to_tcp_ctrl(ctrl)); 1935 if (ret) 1936 goto out_free_queue; 1937 1938 return 0; 1939 1940 out_free_queue: 1941 nvme_tcp_free_queue(ctrl, 0); 1942 return ret; 1943 } 1944 1945 static int __nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl) 1946 { 1947 int i, ret; 1948 1949 if (nvme_tcp_tls(ctrl) && !ctrl->tls_key) { 1950 dev_err(ctrl->device, "no PSK negotiated\n"); 1951 return -ENOKEY; 1952 } 1953 for (i = 1; i < ctrl->queue_count; i++) { 1954 ret = nvme_tcp_alloc_queue(ctrl, i, 1955 key_serial(ctrl->tls_key)); 1956 if (ret) 1957 goto out_free_queues; 1958 } 1959 1960 return 0; 1961 1962 out_free_queues: 1963 for (i--; i >= 1; i--) 1964 nvme_tcp_free_queue(ctrl, i); 1965 1966 return ret; 1967 } 1968 1969 static int nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl) 1970 { 1971 unsigned int nr_io_queues; 1972 int ret; 1973 1974 nr_io_queues = nvmf_nr_io_queues(ctrl->opts); 1975 ret = nvme_set_queue_count(ctrl, &nr_io_queues); 1976 if (ret) 1977 return ret; 1978 1979 if (nr_io_queues == 0) { 1980 dev_err(ctrl->device, 1981 "unable to set any I/O queues\n"); 1982 return -ENOMEM; 1983 } 1984 1985 ctrl->queue_count = nr_io_queues + 1; 1986 dev_info(ctrl->device, 1987 "creating %d I/O queues.\n", nr_io_queues); 1988 1989 nvmf_set_io_queues(ctrl->opts, nr_io_queues, 1990 to_tcp_ctrl(ctrl)->io_queues); 1991 return __nvme_tcp_alloc_io_queues(ctrl); 1992 } 1993 1994 static void nvme_tcp_destroy_io_queues(struct nvme_ctrl *ctrl, bool remove) 1995 { 1996 nvme_tcp_stop_io_queues(ctrl); 1997 if (remove) 1998 nvme_remove_io_tag_set(ctrl); 1999 nvme_tcp_free_io_queues(ctrl); 2000 } 2001 2002 static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new) 2003 { 2004 int ret, nr_queues; 2005 2006 ret = nvme_tcp_alloc_io_queues(ctrl); 2007 if (ret) 2008 return ret; 2009 2010 if (new) { 2011 ret = nvme_alloc_io_tag_set(ctrl, &to_tcp_ctrl(ctrl)->tag_set, 2012 &nvme_tcp_mq_ops, 2013 ctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2, 2014 sizeof(struct nvme_tcp_request)); 2015 if (ret) 2016 goto out_free_io_queues; 2017 } 2018 2019 /* 2020 * Only start IO queues for which we have allocated the tagset 2021 * and limitted it to the available queues. On reconnects, the 2022 * queue number might have changed. 2023 */ 2024 nr_queues = min(ctrl->tagset->nr_hw_queues + 1, ctrl->queue_count); 2025 ret = nvme_tcp_start_io_queues(ctrl, 1, nr_queues); 2026 if (ret) 2027 goto out_cleanup_connect_q; 2028 2029 if (!new) { 2030 nvme_start_freeze(ctrl); 2031 nvme_unquiesce_io_queues(ctrl); 2032 if (!nvme_wait_freeze_timeout(ctrl, NVME_IO_TIMEOUT)) { 2033 /* 2034 * If we timed out waiting for freeze we are likely to 2035 * be stuck. Fail the controller initialization just 2036 * to be safe. 2037 */ 2038 ret = -ENODEV; 2039 nvme_unfreeze(ctrl); 2040 goto out_wait_freeze_timed_out; 2041 } 2042 blk_mq_update_nr_hw_queues(ctrl->tagset, 2043 ctrl->queue_count - 1); 2044 nvme_unfreeze(ctrl); 2045 } 2046 2047 /* 2048 * If the number of queues has increased (reconnect case) 2049 * start all new queues now. 2050 */ 2051 ret = nvme_tcp_start_io_queues(ctrl, nr_queues, 2052 ctrl->tagset->nr_hw_queues + 1); 2053 if (ret) 2054 goto out_wait_freeze_timed_out; 2055 2056 return 0; 2057 2058 out_wait_freeze_timed_out: 2059 nvme_quiesce_io_queues(ctrl); 2060 nvme_sync_io_queues(ctrl); 2061 nvme_tcp_stop_io_queues(ctrl); 2062 out_cleanup_connect_q: 2063 nvme_cancel_tagset(ctrl); 2064 if (new) 2065 nvme_remove_io_tag_set(ctrl); 2066 out_free_io_queues: 2067 nvme_tcp_free_io_queues(ctrl); 2068 return ret; 2069 } 2070 2071 static void nvme_tcp_destroy_admin_queue(struct nvme_ctrl *ctrl, bool remove) 2072 { 2073 nvme_tcp_stop_queue(ctrl, 0); 2074 if (remove) 2075 nvme_remove_admin_tag_set(ctrl); 2076 nvme_tcp_free_admin_queue(ctrl); 2077 } 2078 2079 static int nvme_tcp_configure_admin_queue(struct nvme_ctrl *ctrl, bool new) 2080 { 2081 int error; 2082 2083 error = nvme_tcp_alloc_admin_queue(ctrl); 2084 if (error) 2085 return error; 2086 2087 if (new) { 2088 error = nvme_alloc_admin_tag_set(ctrl, 2089 &to_tcp_ctrl(ctrl)->admin_tag_set, 2090 &nvme_tcp_admin_mq_ops, 2091 sizeof(struct nvme_tcp_request)); 2092 if (error) 2093 goto out_free_queue; 2094 } 2095 2096 error = nvme_tcp_start_queue(ctrl, 0); 2097 if (error) 2098 goto out_cleanup_tagset; 2099 2100 error = nvme_enable_ctrl(ctrl); 2101 if (error) 2102 goto out_stop_queue; 2103 2104 nvme_unquiesce_admin_queue(ctrl); 2105 2106 error = nvme_init_ctrl_finish(ctrl, false); 2107 if (error) 2108 goto out_quiesce_queue; 2109 2110 return 0; 2111 2112 out_quiesce_queue: 2113 nvme_quiesce_admin_queue(ctrl); 2114 blk_sync_queue(ctrl->admin_q); 2115 out_stop_queue: 2116 nvme_tcp_stop_queue(ctrl, 0); 2117 nvme_cancel_admin_tagset(ctrl); 2118 out_cleanup_tagset: 2119 if (new) 2120 nvme_remove_admin_tag_set(ctrl); 2121 out_free_queue: 2122 nvme_tcp_free_admin_queue(ctrl); 2123 return error; 2124 } 2125 2126 static void nvme_tcp_teardown_admin_queue(struct nvme_ctrl *ctrl, 2127 bool remove) 2128 { 2129 nvme_quiesce_admin_queue(ctrl); 2130 blk_sync_queue(ctrl->admin_q); 2131 nvme_tcp_stop_queue(ctrl, 0); 2132 nvme_cancel_admin_tagset(ctrl); 2133 if (remove) 2134 nvme_unquiesce_admin_queue(ctrl); 2135 nvme_tcp_destroy_admin_queue(ctrl, remove); 2136 } 2137 2138 static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl, 2139 bool remove) 2140 { 2141 if (ctrl->queue_count <= 1) 2142 return; 2143 nvme_quiesce_admin_queue(ctrl); 2144 nvme_quiesce_io_queues(ctrl); 2145 nvme_sync_io_queues(ctrl); 2146 nvme_tcp_stop_io_queues(ctrl); 2147 nvme_cancel_tagset(ctrl); 2148 if (remove) 2149 nvme_unquiesce_io_queues(ctrl); 2150 nvme_tcp_destroy_io_queues(ctrl, remove); 2151 } 2152 2153 static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl) 2154 { 2155 /* If we are resetting/deleting then do nothing */ 2156 if (ctrl->state != NVME_CTRL_CONNECTING) { 2157 WARN_ON_ONCE(ctrl->state == NVME_CTRL_NEW || 2158 ctrl->state == NVME_CTRL_LIVE); 2159 return; 2160 } 2161 2162 if (nvmf_should_reconnect(ctrl)) { 2163 dev_info(ctrl->device, "Reconnecting in %d seconds...\n", 2164 ctrl->opts->reconnect_delay); 2165 queue_delayed_work(nvme_wq, &to_tcp_ctrl(ctrl)->connect_work, 2166 ctrl->opts->reconnect_delay * HZ); 2167 } else { 2168 dev_info(ctrl->device, "Removing controller...\n"); 2169 nvme_delete_ctrl(ctrl); 2170 } 2171 } 2172 2173 static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new) 2174 { 2175 struct nvmf_ctrl_options *opts = ctrl->opts; 2176 int ret; 2177 2178 ret = nvme_tcp_configure_admin_queue(ctrl, new); 2179 if (ret) 2180 return ret; 2181 2182 if (ctrl->icdoff) { 2183 ret = -EOPNOTSUPP; 2184 dev_err(ctrl->device, "icdoff is not supported!\n"); 2185 goto destroy_admin; 2186 } 2187 2188 if (!nvme_ctrl_sgl_supported(ctrl)) { 2189 ret = -EOPNOTSUPP; 2190 dev_err(ctrl->device, "Mandatory sgls are not supported!\n"); 2191 goto destroy_admin; 2192 } 2193 2194 if (opts->queue_size > ctrl->sqsize + 1) 2195 dev_warn(ctrl->device, 2196 "queue_size %zu > ctrl sqsize %u, clamping down\n", 2197 opts->queue_size, ctrl->sqsize + 1); 2198 2199 if (ctrl->sqsize + 1 > ctrl->maxcmd) { 2200 dev_warn(ctrl->device, 2201 "sqsize %u > ctrl maxcmd %u, clamping down\n", 2202 ctrl->sqsize + 1, ctrl->maxcmd); 2203 ctrl->sqsize = ctrl->maxcmd - 1; 2204 } 2205 2206 if (ctrl->queue_count > 1) { 2207 ret = nvme_tcp_configure_io_queues(ctrl, new); 2208 if (ret) 2209 goto destroy_admin; 2210 } 2211 2212 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE)) { 2213 /* 2214 * state change failure is ok if we started ctrl delete, 2215 * unless we're during creation of a new controller to 2216 * avoid races with teardown flow. 2217 */ 2218 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING && 2219 ctrl->state != NVME_CTRL_DELETING_NOIO); 2220 WARN_ON_ONCE(new); 2221 ret = -EINVAL; 2222 goto destroy_io; 2223 } 2224 2225 nvme_start_ctrl(ctrl); 2226 return 0; 2227 2228 destroy_io: 2229 if (ctrl->queue_count > 1) { 2230 nvme_quiesce_io_queues(ctrl); 2231 nvme_sync_io_queues(ctrl); 2232 nvme_tcp_stop_io_queues(ctrl); 2233 nvme_cancel_tagset(ctrl); 2234 nvme_tcp_destroy_io_queues(ctrl, new); 2235 } 2236 destroy_admin: 2237 nvme_stop_keep_alive(ctrl); 2238 nvme_tcp_teardown_admin_queue(ctrl, false); 2239 return ret; 2240 } 2241 2242 static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work) 2243 { 2244 struct nvme_tcp_ctrl *tcp_ctrl = container_of(to_delayed_work(work), 2245 struct nvme_tcp_ctrl, connect_work); 2246 struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl; 2247 2248 ++ctrl->nr_reconnects; 2249 2250 if (nvme_tcp_setup_ctrl(ctrl, false)) 2251 goto requeue; 2252 2253 dev_info(ctrl->device, "Successfully reconnected (%d attempt)\n", 2254 ctrl->nr_reconnects); 2255 2256 ctrl->nr_reconnects = 0; 2257 2258 return; 2259 2260 requeue: 2261 dev_info(ctrl->device, "Failed reconnect attempt %d\n", 2262 ctrl->nr_reconnects); 2263 nvme_tcp_reconnect_or_remove(ctrl); 2264 } 2265 2266 static void nvme_tcp_error_recovery_work(struct work_struct *work) 2267 { 2268 struct nvme_tcp_ctrl *tcp_ctrl = container_of(work, 2269 struct nvme_tcp_ctrl, err_work); 2270 struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl; 2271 2272 nvme_stop_keep_alive(ctrl); 2273 flush_work(&ctrl->async_event_work); 2274 nvme_tcp_teardown_io_queues(ctrl, false); 2275 /* unquiesce to fail fast pending requests */ 2276 nvme_unquiesce_io_queues(ctrl); 2277 nvme_tcp_teardown_admin_queue(ctrl, false); 2278 nvme_unquiesce_admin_queue(ctrl); 2279 nvme_auth_stop(ctrl); 2280 2281 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) { 2282 /* state change failure is ok if we started ctrl delete */ 2283 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING && 2284 ctrl->state != NVME_CTRL_DELETING_NOIO); 2285 return; 2286 } 2287 2288 nvme_tcp_reconnect_or_remove(ctrl); 2289 } 2290 2291 static void nvme_tcp_teardown_ctrl(struct nvme_ctrl *ctrl, bool shutdown) 2292 { 2293 nvme_tcp_teardown_io_queues(ctrl, shutdown); 2294 nvme_quiesce_admin_queue(ctrl); 2295 nvme_disable_ctrl(ctrl, shutdown); 2296 nvme_tcp_teardown_admin_queue(ctrl, shutdown); 2297 } 2298 2299 static void nvme_tcp_delete_ctrl(struct nvme_ctrl *ctrl) 2300 { 2301 nvme_tcp_teardown_ctrl(ctrl, true); 2302 } 2303 2304 static void nvme_reset_ctrl_work(struct work_struct *work) 2305 { 2306 struct nvme_ctrl *ctrl = 2307 container_of(work, struct nvme_ctrl, reset_work); 2308 2309 nvme_stop_ctrl(ctrl); 2310 nvme_tcp_teardown_ctrl(ctrl, false); 2311 2312 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) { 2313 /* state change failure is ok if we started ctrl delete */ 2314 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING && 2315 ctrl->state != NVME_CTRL_DELETING_NOIO); 2316 return; 2317 } 2318 2319 if (nvme_tcp_setup_ctrl(ctrl, false)) 2320 goto out_fail; 2321 2322 return; 2323 2324 out_fail: 2325 ++ctrl->nr_reconnects; 2326 nvme_tcp_reconnect_or_remove(ctrl); 2327 } 2328 2329 static void nvme_tcp_stop_ctrl(struct nvme_ctrl *ctrl) 2330 { 2331 flush_work(&to_tcp_ctrl(ctrl)->err_work); 2332 cancel_delayed_work_sync(&to_tcp_ctrl(ctrl)->connect_work); 2333 } 2334 2335 static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl) 2336 { 2337 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 2338 2339 if (list_empty(&ctrl->list)) 2340 goto free_ctrl; 2341 2342 mutex_lock(&nvme_tcp_ctrl_mutex); 2343 list_del(&ctrl->list); 2344 mutex_unlock(&nvme_tcp_ctrl_mutex); 2345 2346 nvmf_free_options(nctrl->opts); 2347 free_ctrl: 2348 kfree(ctrl->queues); 2349 kfree(ctrl); 2350 } 2351 2352 static void nvme_tcp_set_sg_null(struct nvme_command *c) 2353 { 2354 struct nvme_sgl_desc *sg = &c->common.dptr.sgl; 2355 2356 sg->addr = 0; 2357 sg->length = 0; 2358 sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) | 2359 NVME_SGL_FMT_TRANSPORT_A; 2360 } 2361 2362 static void nvme_tcp_set_sg_inline(struct nvme_tcp_queue *queue, 2363 struct nvme_command *c, u32 data_len) 2364 { 2365 struct nvme_sgl_desc *sg = &c->common.dptr.sgl; 2366 2367 sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff); 2368 sg->length = cpu_to_le32(data_len); 2369 sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET; 2370 } 2371 2372 static void nvme_tcp_set_sg_host_data(struct nvme_command *c, 2373 u32 data_len) 2374 { 2375 struct nvme_sgl_desc *sg = &c->common.dptr.sgl; 2376 2377 sg->addr = 0; 2378 sg->length = cpu_to_le32(data_len); 2379 sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) | 2380 NVME_SGL_FMT_TRANSPORT_A; 2381 } 2382 2383 static void nvme_tcp_submit_async_event(struct nvme_ctrl *arg) 2384 { 2385 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(arg); 2386 struct nvme_tcp_queue *queue = &ctrl->queues[0]; 2387 struct nvme_tcp_cmd_pdu *pdu = ctrl->async_req.pdu; 2388 struct nvme_command *cmd = &pdu->cmd; 2389 u8 hdgst = nvme_tcp_hdgst_len(queue); 2390 2391 memset(pdu, 0, sizeof(*pdu)); 2392 pdu->hdr.type = nvme_tcp_cmd; 2393 if (queue->hdr_digest) 2394 pdu->hdr.flags |= NVME_TCP_F_HDGST; 2395 pdu->hdr.hlen = sizeof(*pdu); 2396 pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst); 2397 2398 cmd->common.opcode = nvme_admin_async_event; 2399 cmd->common.command_id = NVME_AQ_BLK_MQ_DEPTH; 2400 cmd->common.flags |= NVME_CMD_SGL_METABUF; 2401 nvme_tcp_set_sg_null(cmd); 2402 2403 ctrl->async_req.state = NVME_TCP_SEND_CMD_PDU; 2404 ctrl->async_req.offset = 0; 2405 ctrl->async_req.curr_bio = NULL; 2406 ctrl->async_req.data_len = 0; 2407 2408 nvme_tcp_queue_request(&ctrl->async_req, true, true); 2409 } 2410 2411 static void nvme_tcp_complete_timed_out(struct request *rq) 2412 { 2413 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2414 struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl; 2415 2416 nvme_tcp_stop_queue(ctrl, nvme_tcp_queue_id(req->queue)); 2417 nvmf_complete_timed_out_request(rq); 2418 } 2419 2420 static enum blk_eh_timer_return nvme_tcp_timeout(struct request *rq) 2421 { 2422 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2423 struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl; 2424 struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req); 2425 u8 opc = pdu->cmd.common.opcode, fctype = pdu->cmd.fabrics.fctype; 2426 int qid = nvme_tcp_queue_id(req->queue); 2427 2428 dev_warn(ctrl->device, 2429 "queue %d: timeout cid %#x type %d opcode %#x (%s)\n", 2430 nvme_tcp_queue_id(req->queue), nvme_cid(rq), pdu->hdr.type, 2431 opc, nvme_opcode_str(qid, opc, fctype)); 2432 2433 if (ctrl->state != NVME_CTRL_LIVE) { 2434 /* 2435 * If we are resetting, connecting or deleting we should 2436 * complete immediately because we may block controller 2437 * teardown or setup sequence 2438 * - ctrl disable/shutdown fabrics requests 2439 * - connect requests 2440 * - initialization admin requests 2441 * - I/O requests that entered after unquiescing and 2442 * the controller stopped responding 2443 * 2444 * All other requests should be cancelled by the error 2445 * recovery work, so it's fine that we fail it here. 2446 */ 2447 nvme_tcp_complete_timed_out(rq); 2448 return BLK_EH_DONE; 2449 } 2450 2451 /* 2452 * LIVE state should trigger the normal error recovery which will 2453 * handle completing this request. 2454 */ 2455 nvme_tcp_error_recovery(ctrl); 2456 return BLK_EH_RESET_TIMER; 2457 } 2458 2459 static blk_status_t nvme_tcp_map_data(struct nvme_tcp_queue *queue, 2460 struct request *rq) 2461 { 2462 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2463 struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req); 2464 struct nvme_command *c = &pdu->cmd; 2465 2466 c->common.flags |= NVME_CMD_SGL_METABUF; 2467 2468 if (!blk_rq_nr_phys_segments(rq)) 2469 nvme_tcp_set_sg_null(c); 2470 else if (rq_data_dir(rq) == WRITE && 2471 req->data_len <= nvme_tcp_inline_data_size(req)) 2472 nvme_tcp_set_sg_inline(queue, c, req->data_len); 2473 else 2474 nvme_tcp_set_sg_host_data(c, req->data_len); 2475 2476 return 0; 2477 } 2478 2479 static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns, 2480 struct request *rq) 2481 { 2482 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2483 struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req); 2484 struct nvme_tcp_queue *queue = req->queue; 2485 u8 hdgst = nvme_tcp_hdgst_len(queue), ddgst = 0; 2486 blk_status_t ret; 2487 2488 ret = nvme_setup_cmd(ns, rq); 2489 if (ret) 2490 return ret; 2491 2492 req->state = NVME_TCP_SEND_CMD_PDU; 2493 req->status = cpu_to_le16(NVME_SC_SUCCESS); 2494 req->offset = 0; 2495 req->data_sent = 0; 2496 req->pdu_len = 0; 2497 req->pdu_sent = 0; 2498 req->h2cdata_left = 0; 2499 req->data_len = blk_rq_nr_phys_segments(rq) ? 2500 blk_rq_payload_bytes(rq) : 0; 2501 req->curr_bio = rq->bio; 2502 if (req->curr_bio && req->data_len) 2503 nvme_tcp_init_iter(req, rq_data_dir(rq)); 2504 2505 if (rq_data_dir(rq) == WRITE && 2506 req->data_len <= nvme_tcp_inline_data_size(req)) 2507 req->pdu_len = req->data_len; 2508 2509 pdu->hdr.type = nvme_tcp_cmd; 2510 pdu->hdr.flags = 0; 2511 if (queue->hdr_digest) 2512 pdu->hdr.flags |= NVME_TCP_F_HDGST; 2513 if (queue->data_digest && req->pdu_len) { 2514 pdu->hdr.flags |= NVME_TCP_F_DDGST; 2515 ddgst = nvme_tcp_ddgst_len(queue); 2516 } 2517 pdu->hdr.hlen = sizeof(*pdu); 2518 pdu->hdr.pdo = req->pdu_len ? pdu->hdr.hlen + hdgst : 0; 2519 pdu->hdr.plen = 2520 cpu_to_le32(pdu->hdr.hlen + hdgst + req->pdu_len + ddgst); 2521 2522 ret = nvme_tcp_map_data(queue, rq); 2523 if (unlikely(ret)) { 2524 nvme_cleanup_cmd(rq); 2525 dev_err(queue->ctrl->ctrl.device, 2526 "Failed to map data (%d)\n", ret); 2527 return ret; 2528 } 2529 2530 return 0; 2531 } 2532 2533 static void nvme_tcp_commit_rqs(struct blk_mq_hw_ctx *hctx) 2534 { 2535 struct nvme_tcp_queue *queue = hctx->driver_data; 2536 2537 if (!llist_empty(&queue->req_list)) 2538 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 2539 } 2540 2541 static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx, 2542 const struct blk_mq_queue_data *bd) 2543 { 2544 struct nvme_ns *ns = hctx->queue->queuedata; 2545 struct nvme_tcp_queue *queue = hctx->driver_data; 2546 struct request *rq = bd->rq; 2547 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2548 bool queue_ready = test_bit(NVME_TCP_Q_LIVE, &queue->flags); 2549 blk_status_t ret; 2550 2551 if (!nvme_check_ready(&queue->ctrl->ctrl, rq, queue_ready)) 2552 return nvme_fail_nonready_command(&queue->ctrl->ctrl, rq); 2553 2554 ret = nvme_tcp_setup_cmd_pdu(ns, rq); 2555 if (unlikely(ret)) 2556 return ret; 2557 2558 nvme_start_request(rq); 2559 2560 nvme_tcp_queue_request(req, true, bd->last); 2561 2562 return BLK_STS_OK; 2563 } 2564 2565 static void nvme_tcp_map_queues(struct blk_mq_tag_set *set) 2566 { 2567 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(set->driver_data); 2568 2569 nvmf_map_queues(set, &ctrl->ctrl, ctrl->io_queues); 2570 } 2571 2572 static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) 2573 { 2574 struct nvme_tcp_queue *queue = hctx->driver_data; 2575 struct sock *sk = queue->sock->sk; 2576 2577 if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags)) 2578 return 0; 2579 2580 set_bit(NVME_TCP_Q_POLLING, &queue->flags); 2581 if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(&sk->sk_receive_queue)) 2582 sk_busy_loop(sk, true); 2583 nvme_tcp_try_recv(queue); 2584 clear_bit(NVME_TCP_Q_POLLING, &queue->flags); 2585 return queue->nr_cqe; 2586 } 2587 2588 static int nvme_tcp_get_address(struct nvme_ctrl *ctrl, char *buf, int size) 2589 { 2590 struct nvme_tcp_queue *queue = &to_tcp_ctrl(ctrl)->queues[0]; 2591 struct sockaddr_storage src_addr; 2592 int ret, len; 2593 2594 len = nvmf_get_address(ctrl, buf, size); 2595 2596 mutex_lock(&queue->queue_lock); 2597 2598 if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags)) 2599 goto done; 2600 ret = kernel_getsockname(queue->sock, (struct sockaddr *)&src_addr); 2601 if (ret > 0) { 2602 if (len > 0) 2603 len--; /* strip trailing newline */ 2604 len += scnprintf(buf + len, size - len, "%ssrc_addr=%pISc\n", 2605 (len) ? "," : "", &src_addr); 2606 } 2607 done: 2608 mutex_unlock(&queue->queue_lock); 2609 2610 return len; 2611 } 2612 2613 static const struct blk_mq_ops nvme_tcp_mq_ops = { 2614 .queue_rq = nvme_tcp_queue_rq, 2615 .commit_rqs = nvme_tcp_commit_rqs, 2616 .complete = nvme_complete_rq, 2617 .init_request = nvme_tcp_init_request, 2618 .exit_request = nvme_tcp_exit_request, 2619 .init_hctx = nvme_tcp_init_hctx, 2620 .timeout = nvme_tcp_timeout, 2621 .map_queues = nvme_tcp_map_queues, 2622 .poll = nvme_tcp_poll, 2623 }; 2624 2625 static const struct blk_mq_ops nvme_tcp_admin_mq_ops = { 2626 .queue_rq = nvme_tcp_queue_rq, 2627 .complete = nvme_complete_rq, 2628 .init_request = nvme_tcp_init_request, 2629 .exit_request = nvme_tcp_exit_request, 2630 .init_hctx = nvme_tcp_init_admin_hctx, 2631 .timeout = nvme_tcp_timeout, 2632 }; 2633 2634 static const struct nvme_ctrl_ops nvme_tcp_ctrl_ops = { 2635 .name = "tcp", 2636 .module = THIS_MODULE, 2637 .flags = NVME_F_FABRICS | NVME_F_BLOCKING, 2638 .reg_read32 = nvmf_reg_read32, 2639 .reg_read64 = nvmf_reg_read64, 2640 .reg_write32 = nvmf_reg_write32, 2641 .free_ctrl = nvme_tcp_free_ctrl, 2642 .submit_async_event = nvme_tcp_submit_async_event, 2643 .delete_ctrl = nvme_tcp_delete_ctrl, 2644 .get_address = nvme_tcp_get_address, 2645 .stop_ctrl = nvme_tcp_stop_ctrl, 2646 }; 2647 2648 static bool 2649 nvme_tcp_existing_controller(struct nvmf_ctrl_options *opts) 2650 { 2651 struct nvme_tcp_ctrl *ctrl; 2652 bool found = false; 2653 2654 mutex_lock(&nvme_tcp_ctrl_mutex); 2655 list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) { 2656 found = nvmf_ip_options_match(&ctrl->ctrl, opts); 2657 if (found) 2658 break; 2659 } 2660 mutex_unlock(&nvme_tcp_ctrl_mutex); 2661 2662 return found; 2663 } 2664 2665 static struct nvme_ctrl *nvme_tcp_create_ctrl(struct device *dev, 2666 struct nvmf_ctrl_options *opts) 2667 { 2668 struct nvme_tcp_ctrl *ctrl; 2669 int ret; 2670 2671 ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL); 2672 if (!ctrl) 2673 return ERR_PTR(-ENOMEM); 2674 2675 INIT_LIST_HEAD(&ctrl->list); 2676 ctrl->ctrl.opts = opts; 2677 ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues + 2678 opts->nr_poll_queues + 1; 2679 ctrl->ctrl.sqsize = opts->queue_size - 1; 2680 ctrl->ctrl.kato = opts->kato; 2681 2682 INIT_DELAYED_WORK(&ctrl->connect_work, 2683 nvme_tcp_reconnect_ctrl_work); 2684 INIT_WORK(&ctrl->err_work, nvme_tcp_error_recovery_work); 2685 INIT_WORK(&ctrl->ctrl.reset_work, nvme_reset_ctrl_work); 2686 2687 if (!(opts->mask & NVMF_OPT_TRSVCID)) { 2688 opts->trsvcid = 2689 kstrdup(__stringify(NVME_TCP_DISC_PORT), GFP_KERNEL); 2690 if (!opts->trsvcid) { 2691 ret = -ENOMEM; 2692 goto out_free_ctrl; 2693 } 2694 opts->mask |= NVMF_OPT_TRSVCID; 2695 } 2696 2697 ret = inet_pton_with_scope(&init_net, AF_UNSPEC, 2698 opts->traddr, opts->trsvcid, &ctrl->addr); 2699 if (ret) { 2700 pr_err("malformed address passed: %s:%s\n", 2701 opts->traddr, opts->trsvcid); 2702 goto out_free_ctrl; 2703 } 2704 2705 if (opts->mask & NVMF_OPT_HOST_TRADDR) { 2706 ret = inet_pton_with_scope(&init_net, AF_UNSPEC, 2707 opts->host_traddr, NULL, &ctrl->src_addr); 2708 if (ret) { 2709 pr_err("malformed src address passed: %s\n", 2710 opts->host_traddr); 2711 goto out_free_ctrl; 2712 } 2713 } 2714 2715 if (opts->mask & NVMF_OPT_HOST_IFACE) { 2716 if (!__dev_get_by_name(&init_net, opts->host_iface)) { 2717 pr_err("invalid interface passed: %s\n", 2718 opts->host_iface); 2719 ret = -ENODEV; 2720 goto out_free_ctrl; 2721 } 2722 } 2723 2724 if (!opts->duplicate_connect && nvme_tcp_existing_controller(opts)) { 2725 ret = -EALREADY; 2726 goto out_free_ctrl; 2727 } 2728 2729 ctrl->queues = kcalloc(ctrl->ctrl.queue_count, sizeof(*ctrl->queues), 2730 GFP_KERNEL); 2731 if (!ctrl->queues) { 2732 ret = -ENOMEM; 2733 goto out_free_ctrl; 2734 } 2735 2736 ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_tcp_ctrl_ops, 0); 2737 if (ret) 2738 goto out_kfree_queues; 2739 2740 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) { 2741 WARN_ON_ONCE(1); 2742 ret = -EINTR; 2743 goto out_uninit_ctrl; 2744 } 2745 2746 ret = nvme_tcp_setup_ctrl(&ctrl->ctrl, true); 2747 if (ret) 2748 goto out_uninit_ctrl; 2749 2750 dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISp\n", 2751 nvmf_ctrl_subsysnqn(&ctrl->ctrl), &ctrl->addr); 2752 2753 mutex_lock(&nvme_tcp_ctrl_mutex); 2754 list_add_tail(&ctrl->list, &nvme_tcp_ctrl_list); 2755 mutex_unlock(&nvme_tcp_ctrl_mutex); 2756 2757 return &ctrl->ctrl; 2758 2759 out_uninit_ctrl: 2760 nvme_uninit_ctrl(&ctrl->ctrl); 2761 nvme_put_ctrl(&ctrl->ctrl); 2762 if (ret > 0) 2763 ret = -EIO; 2764 return ERR_PTR(ret); 2765 out_kfree_queues: 2766 kfree(ctrl->queues); 2767 out_free_ctrl: 2768 kfree(ctrl); 2769 return ERR_PTR(ret); 2770 } 2771 2772 static struct nvmf_transport_ops nvme_tcp_transport = { 2773 .name = "tcp", 2774 .module = THIS_MODULE, 2775 .required_opts = NVMF_OPT_TRADDR, 2776 .allowed_opts = NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY | 2777 NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO | 2778 NVMF_OPT_HDR_DIGEST | NVMF_OPT_DATA_DIGEST | 2779 NVMF_OPT_NR_WRITE_QUEUES | NVMF_OPT_NR_POLL_QUEUES | 2780 NVMF_OPT_TOS | NVMF_OPT_HOST_IFACE | NVMF_OPT_TLS | 2781 NVMF_OPT_KEYRING | NVMF_OPT_TLS_KEY, 2782 .create_ctrl = nvme_tcp_create_ctrl, 2783 }; 2784 2785 static int __init nvme_tcp_init_module(void) 2786 { 2787 BUILD_BUG_ON(sizeof(struct nvme_tcp_hdr) != 8); 2788 BUILD_BUG_ON(sizeof(struct nvme_tcp_cmd_pdu) != 72); 2789 BUILD_BUG_ON(sizeof(struct nvme_tcp_data_pdu) != 24); 2790 BUILD_BUG_ON(sizeof(struct nvme_tcp_rsp_pdu) != 24); 2791 BUILD_BUG_ON(sizeof(struct nvme_tcp_r2t_pdu) != 24); 2792 BUILD_BUG_ON(sizeof(struct nvme_tcp_icreq_pdu) != 128); 2793 BUILD_BUG_ON(sizeof(struct nvme_tcp_icresp_pdu) != 128); 2794 BUILD_BUG_ON(sizeof(struct nvme_tcp_term_pdu) != 24); 2795 2796 nvme_tcp_wq = alloc_workqueue("nvme_tcp_wq", 2797 WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); 2798 if (!nvme_tcp_wq) 2799 return -ENOMEM; 2800 2801 nvmf_register_transport(&nvme_tcp_transport); 2802 return 0; 2803 } 2804 2805 static void __exit nvme_tcp_cleanup_module(void) 2806 { 2807 struct nvme_tcp_ctrl *ctrl; 2808 2809 nvmf_unregister_transport(&nvme_tcp_transport); 2810 2811 mutex_lock(&nvme_tcp_ctrl_mutex); 2812 list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) 2813 nvme_delete_ctrl(&ctrl->ctrl); 2814 mutex_unlock(&nvme_tcp_ctrl_mutex); 2815 flush_workqueue(nvme_delete_wq); 2816 2817 destroy_workqueue(nvme_tcp_wq); 2818 } 2819 2820 module_init(nvme_tcp_init_module); 2821 module_exit(nvme_tcp_cleanup_module); 2822 2823 MODULE_LICENSE("GPL v2"); 2824