1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2017, Microsoft Corporation. 4 * Copyright (C) 2018, LG Electronics. 5 * 6 * Author(s): Long Li <longli@microsoft.com>, 7 * Hyunchul Lee <hyc.lee@gmail.com> 8 */ 9 10 #define SUBMOD_NAME "smb_direct" 11 12 #include <linux/kthread.h> 13 #include <linux/list.h> 14 #include <linux/mempool.h> 15 #include <linux/highmem.h> 16 #include <linux/scatterlist.h> 17 #include <linux/string_choices.h> 18 #include <rdma/ib_verbs.h> 19 #include <rdma/rdma_cm.h> 20 #include <rdma/rw.h> 21 22 #include "glob.h" 23 #include "connection.h" 24 #include "smb_common.h" 25 #include "../common/smb2status.h" 26 #include "../common/smbdirect/smbdirect.h" 27 #include "../common/smbdirect/smbdirect_pdu.h" 28 #include "../common/smbdirect/smbdirect_socket.h" 29 #include "transport_rdma.h" 30 31 #define SMB_DIRECT_PORT_IWARP 5445 32 #define SMB_DIRECT_PORT_INFINIBAND 445 33 34 #define SMB_DIRECT_VERSION_LE cpu_to_le16(SMBDIRECT_V1) 35 36 /* SMB_DIRECT negotiation timeout (for the server) in seconds */ 37 #define SMB_DIRECT_NEGOTIATE_TIMEOUT 5 38 39 /* The timeout to wait for a keepalive message from peer in seconds */ 40 #define SMB_DIRECT_KEEPALIVE_SEND_INTERVAL 120 41 42 /* The timeout to wait for a keepalive message from peer in seconds */ 43 #define SMB_DIRECT_KEEPALIVE_RECV_TIMEOUT 5 44 45 /* 46 * Default maximum number of RDMA read/write outstanding on this connection 47 * This value is possibly decreased during QP creation on hardware limit 48 */ 49 #define SMB_DIRECT_CM_INITIATOR_DEPTH 8 50 51 /* Maximum number of retries on data transfer operations */ 52 #define SMB_DIRECT_CM_RETRY 6 53 /* No need to retry on Receiver Not Ready since SMB_DIRECT manages credits */ 54 #define SMB_DIRECT_CM_RNR_RETRY 0 55 56 /* 57 * User configurable initial values per SMB_DIRECT transport connection 58 * as defined in [MS-SMBD] 3.1.1.1 59 * Those may change after a SMB_DIRECT negotiation 60 */ 61 62 /* Set 445 port to SMB Direct port by default */ 63 static int smb_direct_port = SMB_DIRECT_PORT_INFINIBAND; 64 65 /* The local peer's maximum number of credits to grant to the peer */ 66 static int smb_direct_receive_credit_max = 255; 67 68 /* The remote peer's credit request of local peer */ 69 static int smb_direct_send_credit_target = 255; 70 71 /* The maximum single message size can be sent to remote peer */ 72 static int smb_direct_max_send_size = 1364; 73 74 /* The maximum fragmented upper-layer payload receive size supported */ 75 static int smb_direct_max_fragmented_recv_size = 1024 * 1024; 76 77 /* The maximum single-message size which can be received */ 78 static int smb_direct_max_receive_size = 1364; 79 80 static int smb_direct_max_read_write_size = SMBD_DEFAULT_IOSIZE; 81 82 static LIST_HEAD(smb_direct_device_list); 83 static DEFINE_RWLOCK(smb_direct_device_lock); 84 85 struct smb_direct_device { 86 struct ib_device *ib_dev; 87 struct list_head list; 88 }; 89 90 static struct smb_direct_listener { 91 struct rdma_cm_id *cm_id; 92 } smb_direct_listener; 93 94 static struct workqueue_struct *smb_direct_wq; 95 96 struct smb_direct_transport { 97 struct ksmbd_transport transport; 98 99 struct smbdirect_socket socket; 100 }; 101 102 #define KSMBD_TRANS(t) (&(t)->transport) 103 #define SMBD_TRANS(t) (container_of(t, \ 104 struct smb_direct_transport, transport)) 105 106 static const struct ksmbd_transport_ops ksmbd_smb_direct_transport_ops; 107 108 void init_smbd_max_io_size(unsigned int sz) 109 { 110 sz = clamp_val(sz, SMBD_MIN_IOSIZE, SMBD_MAX_IOSIZE); 111 smb_direct_max_read_write_size = sz; 112 } 113 114 unsigned int get_smbd_max_read_write_size(struct ksmbd_transport *kt) 115 { 116 struct smb_direct_transport *t; 117 struct smbdirect_socket *sc; 118 struct smbdirect_socket_parameters *sp; 119 120 if (kt->ops != &ksmbd_smb_direct_transport_ops) 121 return 0; 122 123 t = SMBD_TRANS(kt); 124 sc = &t->socket; 125 sp = &sc->parameters; 126 127 return sp->max_read_write_size; 128 } 129 130 static inline int get_buf_page_count(void *buf, int size) 131 { 132 return DIV_ROUND_UP((uintptr_t)buf + size, PAGE_SIZE) - 133 (uintptr_t)buf / PAGE_SIZE; 134 } 135 136 static void smb_direct_destroy_pools(struct smbdirect_socket *sc); 137 static void smb_direct_post_recv_credits(struct work_struct *work); 138 static int smb_direct_post_send_data(struct smbdirect_socket *sc, 139 struct smbdirect_send_batch *send_ctx, 140 struct kvec *iov, int niov, 141 int remaining_data_length); 142 143 static inline void 144 *smbdirect_recv_io_payload(struct smbdirect_recv_io *recvmsg) 145 { 146 return (void *)recvmsg->packet; 147 } 148 149 static struct 150 smbdirect_recv_io *get_free_recvmsg(struct smbdirect_socket *sc) 151 { 152 struct smbdirect_recv_io *recvmsg = NULL; 153 unsigned long flags; 154 155 spin_lock_irqsave(&sc->recv_io.free.lock, flags); 156 if (!list_empty(&sc->recv_io.free.list)) { 157 recvmsg = list_first_entry(&sc->recv_io.free.list, 158 struct smbdirect_recv_io, 159 list); 160 list_del(&recvmsg->list); 161 } 162 spin_unlock_irqrestore(&sc->recv_io.free.lock, flags); 163 return recvmsg; 164 } 165 166 static void put_recvmsg(struct smbdirect_socket *sc, 167 struct smbdirect_recv_io *recvmsg) 168 { 169 unsigned long flags; 170 171 if (likely(recvmsg->sge.length != 0)) { 172 ib_dma_unmap_single(sc->ib.dev, 173 recvmsg->sge.addr, 174 recvmsg->sge.length, 175 DMA_FROM_DEVICE); 176 recvmsg->sge.length = 0; 177 } 178 179 spin_lock_irqsave(&sc->recv_io.free.lock, flags); 180 list_add(&recvmsg->list, &sc->recv_io.free.list); 181 spin_unlock_irqrestore(&sc->recv_io.free.lock, flags); 182 183 queue_work(sc->workqueue, &sc->recv_io.posted.refill_work); 184 } 185 186 static void enqueue_reassembly(struct smbdirect_socket *sc, 187 struct smbdirect_recv_io *recvmsg, 188 int data_length) 189 { 190 unsigned long flags; 191 192 spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); 193 list_add_tail(&recvmsg->list, &sc->recv_io.reassembly.list); 194 sc->recv_io.reassembly.queue_length++; 195 /* 196 * Make sure reassembly_data_length is updated after list and 197 * reassembly_queue_length are updated. On the dequeue side 198 * reassembly_data_length is checked without a lock to determine 199 * if reassembly_queue_length and list is up to date 200 */ 201 virt_wmb(); 202 sc->recv_io.reassembly.data_length += data_length; 203 spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); 204 } 205 206 static struct smbdirect_recv_io *get_first_reassembly(struct smbdirect_socket *sc) 207 { 208 if (!list_empty(&sc->recv_io.reassembly.list)) 209 return list_first_entry(&sc->recv_io.reassembly.list, 210 struct smbdirect_recv_io, list); 211 else 212 return NULL; 213 } 214 215 static void smb_direct_disconnect_wake_up_all(struct smbdirect_socket *sc) 216 { 217 /* 218 * Wake up all waiters in all wait queues 219 * in order to notice the broken connection. 220 */ 221 wake_up_all(&sc->status_wait); 222 wake_up_all(&sc->send_io.credits.wait_queue); 223 wake_up_all(&sc->send_io.pending.zero_wait_queue); 224 wake_up_all(&sc->recv_io.reassembly.wait_queue); 225 wake_up_all(&sc->rw_io.credits.wait_queue); 226 } 227 228 static void smb_direct_disconnect_rdma_work(struct work_struct *work) 229 { 230 struct smbdirect_socket *sc = 231 container_of(work, struct smbdirect_socket, disconnect_work); 232 233 /* 234 * make sure this and other work is not queued again 235 * but here we don't block and avoid 236 * disable[_delayed]_work_sync() 237 */ 238 disable_work(&sc->disconnect_work); 239 disable_work(&sc->recv_io.posted.refill_work); 240 disable_delayed_work(&sc->idle.timer_work); 241 disable_work(&sc->idle.immediate_work); 242 243 if (sc->first_error == 0) 244 sc->first_error = -ECONNABORTED; 245 246 switch (sc->status) { 247 case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED: 248 case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING: 249 case SMBDIRECT_SOCKET_NEGOTIATE_FAILED: 250 case SMBDIRECT_SOCKET_CONNECTED: 251 case SMBDIRECT_SOCKET_ERROR: 252 sc->status = SMBDIRECT_SOCKET_DISCONNECTING; 253 rdma_disconnect(sc->rdma.cm_id); 254 break; 255 256 case SMBDIRECT_SOCKET_CREATED: 257 case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED: 258 case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING: 259 case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED: 260 case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED: 261 case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING: 262 case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED: 263 case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED: 264 case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING: 265 case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED: 266 /* 267 * rdma_accept() never reached 268 * RDMA_CM_EVENT_ESTABLISHED 269 */ 270 sc->status = SMBDIRECT_SOCKET_DISCONNECTED; 271 break; 272 273 case SMBDIRECT_SOCKET_DISCONNECTING: 274 case SMBDIRECT_SOCKET_DISCONNECTED: 275 case SMBDIRECT_SOCKET_DESTROYED: 276 break; 277 } 278 279 /* 280 * Wake up all waiters in all wait queues 281 * in order to notice the broken connection. 282 */ 283 smb_direct_disconnect_wake_up_all(sc); 284 } 285 286 static void 287 smb_direct_disconnect_rdma_connection(struct smbdirect_socket *sc) 288 { 289 /* 290 * make sure other work (than disconnect_work) is 291 * not queued again but here we don't block and avoid 292 * disable[_delayed]_work_sync() 293 */ 294 disable_work(&sc->recv_io.posted.refill_work); 295 disable_work(&sc->idle.immediate_work); 296 disable_delayed_work(&sc->idle.timer_work); 297 298 if (sc->first_error == 0) 299 sc->first_error = -ECONNABORTED; 300 301 switch (sc->status) { 302 case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED: 303 case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED: 304 case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED: 305 case SMBDIRECT_SOCKET_NEGOTIATE_FAILED: 306 case SMBDIRECT_SOCKET_ERROR: 307 case SMBDIRECT_SOCKET_DISCONNECTING: 308 case SMBDIRECT_SOCKET_DISCONNECTED: 309 case SMBDIRECT_SOCKET_DESTROYED: 310 /* 311 * Keep the current error status 312 */ 313 break; 314 315 case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED: 316 case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING: 317 sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED; 318 break; 319 320 case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED: 321 case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING: 322 sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED; 323 break; 324 325 case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED: 326 case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING: 327 sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED; 328 break; 329 330 case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED: 331 case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING: 332 sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED; 333 break; 334 335 case SMBDIRECT_SOCKET_CREATED: 336 case SMBDIRECT_SOCKET_CONNECTED: 337 sc->status = SMBDIRECT_SOCKET_ERROR; 338 break; 339 } 340 341 /* 342 * Wake up all waiters in all wait queues 343 * in order to notice the broken connection. 344 */ 345 smb_direct_disconnect_wake_up_all(sc); 346 347 queue_work(sc->workqueue, &sc->disconnect_work); 348 } 349 350 static void smb_direct_send_immediate_work(struct work_struct *work) 351 { 352 struct smbdirect_socket *sc = 353 container_of(work, struct smbdirect_socket, idle.immediate_work); 354 355 if (sc->status != SMBDIRECT_SOCKET_CONNECTED) 356 return; 357 358 smb_direct_post_send_data(sc, NULL, NULL, 0, 0); 359 } 360 361 static void smb_direct_idle_connection_timer(struct work_struct *work) 362 { 363 struct smbdirect_socket *sc = 364 container_of(work, struct smbdirect_socket, idle.timer_work.work); 365 struct smbdirect_socket_parameters *sp = &sc->parameters; 366 367 if (sc->idle.keepalive != SMBDIRECT_KEEPALIVE_NONE) { 368 smb_direct_disconnect_rdma_connection(sc); 369 return; 370 } 371 372 if (sc->status != SMBDIRECT_SOCKET_CONNECTED) 373 return; 374 375 /* 376 * Now use the keepalive timeout (instead of keepalive interval) 377 * in order to wait for a response 378 */ 379 sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING; 380 mod_delayed_work(sc->workqueue, &sc->idle.timer_work, 381 msecs_to_jiffies(sp->keepalive_timeout_msec)); 382 queue_work(sc->workqueue, &sc->idle.immediate_work); 383 } 384 385 static struct smb_direct_transport *alloc_transport(struct rdma_cm_id *cm_id) 386 { 387 struct smb_direct_transport *t; 388 struct smbdirect_socket *sc; 389 struct smbdirect_socket_parameters *sp; 390 struct ksmbd_conn *conn; 391 392 t = kzalloc(sizeof(*t), KSMBD_DEFAULT_GFP); 393 if (!t) 394 return NULL; 395 sc = &t->socket; 396 smbdirect_socket_init(sc); 397 sp = &sc->parameters; 398 399 sc->workqueue = smb_direct_wq; 400 401 INIT_WORK(&sc->disconnect_work, smb_direct_disconnect_rdma_work); 402 403 sp->negotiate_timeout_msec = SMB_DIRECT_NEGOTIATE_TIMEOUT * 1000; 404 sp->initiator_depth = SMB_DIRECT_CM_INITIATOR_DEPTH; 405 sp->responder_resources = 1; 406 sp->recv_credit_max = smb_direct_receive_credit_max; 407 sp->send_credit_target = smb_direct_send_credit_target; 408 sp->max_send_size = smb_direct_max_send_size; 409 sp->max_fragmented_recv_size = smb_direct_max_fragmented_recv_size; 410 sp->max_recv_size = smb_direct_max_receive_size; 411 sp->max_read_write_size = smb_direct_max_read_write_size; 412 sp->keepalive_interval_msec = SMB_DIRECT_KEEPALIVE_SEND_INTERVAL * 1000; 413 sp->keepalive_timeout_msec = SMB_DIRECT_KEEPALIVE_RECV_TIMEOUT * 1000; 414 415 sc->rdma.cm_id = cm_id; 416 cm_id->context = sc; 417 418 sc->ib.dev = sc->rdma.cm_id->device; 419 420 INIT_WORK(&sc->recv_io.posted.refill_work, 421 smb_direct_post_recv_credits); 422 INIT_WORK(&sc->idle.immediate_work, smb_direct_send_immediate_work); 423 INIT_DELAYED_WORK(&sc->idle.timer_work, smb_direct_idle_connection_timer); 424 425 conn = ksmbd_conn_alloc(); 426 if (!conn) 427 goto err; 428 429 down_write(&conn_list_lock); 430 hash_add(conn_list, &conn->hlist, 0); 431 up_write(&conn_list_lock); 432 433 conn->transport = KSMBD_TRANS(t); 434 KSMBD_TRANS(t)->conn = conn; 435 KSMBD_TRANS(t)->ops = &ksmbd_smb_direct_transport_ops; 436 return t; 437 err: 438 kfree(t); 439 return NULL; 440 } 441 442 static void smb_direct_free_transport(struct ksmbd_transport *kt) 443 { 444 kfree(SMBD_TRANS(kt)); 445 } 446 447 static void free_transport(struct smb_direct_transport *t) 448 { 449 struct smbdirect_socket *sc = &t->socket; 450 struct smbdirect_recv_io *recvmsg; 451 452 disable_work_sync(&sc->disconnect_work); 453 if (sc->status < SMBDIRECT_SOCKET_DISCONNECTING) { 454 smb_direct_disconnect_rdma_work(&sc->disconnect_work); 455 wait_event_interruptible(sc->status_wait, 456 sc->status == SMBDIRECT_SOCKET_DISCONNECTED); 457 } 458 459 /* 460 * Wake up all waiters in all wait queues 461 * in order to notice the broken connection. 462 * 463 * Most likely this was already called via 464 * smb_direct_disconnect_rdma_work(), but call it again... 465 */ 466 smb_direct_disconnect_wake_up_all(sc); 467 468 disable_work_sync(&sc->recv_io.posted.refill_work); 469 disable_delayed_work_sync(&sc->idle.timer_work); 470 disable_work_sync(&sc->idle.immediate_work); 471 472 if (sc->ib.qp) { 473 ib_drain_qp(sc->ib.qp); 474 ib_mr_pool_destroy(sc->ib.qp, &sc->ib.qp->rdma_mrs); 475 sc->ib.qp = NULL; 476 rdma_destroy_qp(sc->rdma.cm_id); 477 } 478 479 ksmbd_debug(RDMA, "drain the reassembly queue\n"); 480 do { 481 unsigned long flags; 482 483 spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); 484 recvmsg = get_first_reassembly(sc); 485 if (recvmsg) { 486 list_del(&recvmsg->list); 487 spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); 488 put_recvmsg(sc, recvmsg); 489 } else { 490 spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); 491 } 492 } while (recvmsg); 493 sc->recv_io.reassembly.data_length = 0; 494 495 if (sc->ib.send_cq) 496 ib_free_cq(sc->ib.send_cq); 497 if (sc->ib.recv_cq) 498 ib_free_cq(sc->ib.recv_cq); 499 if (sc->ib.pd) 500 ib_dealloc_pd(sc->ib.pd); 501 if (sc->rdma.cm_id) 502 rdma_destroy_id(sc->rdma.cm_id); 503 504 smb_direct_destroy_pools(sc); 505 ksmbd_conn_free(KSMBD_TRANS(t)->conn); 506 } 507 508 static struct smbdirect_send_io 509 *smb_direct_alloc_sendmsg(struct smbdirect_socket *sc) 510 { 511 struct smbdirect_send_io *msg; 512 513 msg = mempool_alloc(sc->send_io.mem.pool, KSMBD_DEFAULT_GFP); 514 if (!msg) 515 return ERR_PTR(-ENOMEM); 516 msg->socket = sc; 517 INIT_LIST_HEAD(&msg->sibling_list); 518 msg->num_sge = 0; 519 return msg; 520 } 521 522 static void smb_direct_free_sendmsg(struct smbdirect_socket *sc, 523 struct smbdirect_send_io *msg) 524 { 525 int i; 526 527 if (msg->num_sge > 0) { 528 ib_dma_unmap_single(sc->ib.dev, 529 msg->sge[0].addr, msg->sge[0].length, 530 DMA_TO_DEVICE); 531 for (i = 1; i < msg->num_sge; i++) 532 ib_dma_unmap_page(sc->ib.dev, 533 msg->sge[i].addr, msg->sge[i].length, 534 DMA_TO_DEVICE); 535 } 536 mempool_free(msg, sc->send_io.mem.pool); 537 } 538 539 static int smb_direct_check_recvmsg(struct smbdirect_recv_io *recvmsg) 540 { 541 struct smbdirect_socket *sc = recvmsg->socket; 542 543 switch (sc->recv_io.expected) { 544 case SMBDIRECT_EXPECT_DATA_TRANSFER: { 545 struct smbdirect_data_transfer *req = 546 (struct smbdirect_data_transfer *)recvmsg->packet; 547 struct smb2_hdr *hdr = (struct smb2_hdr *)(recvmsg->packet 548 + le32_to_cpu(req->data_offset)); 549 ksmbd_debug(RDMA, 550 "CreditGranted: %u, CreditRequested: %u, DataLength: %u, RemainingDataLength: %u, SMB: %x, Command: %u\n", 551 le16_to_cpu(req->credits_granted), 552 le16_to_cpu(req->credits_requested), 553 req->data_length, req->remaining_data_length, 554 hdr->ProtocolId, hdr->Command); 555 return 0; 556 } 557 case SMBDIRECT_EXPECT_NEGOTIATE_REQ: { 558 struct smbdirect_negotiate_req *req = 559 (struct smbdirect_negotiate_req *)recvmsg->packet; 560 ksmbd_debug(RDMA, 561 "MinVersion: %u, MaxVersion: %u, CreditRequested: %u, MaxSendSize: %u, MaxRecvSize: %u, MaxFragmentedSize: %u\n", 562 le16_to_cpu(req->min_version), 563 le16_to_cpu(req->max_version), 564 le16_to_cpu(req->credits_requested), 565 le32_to_cpu(req->preferred_send_size), 566 le32_to_cpu(req->max_receive_size), 567 le32_to_cpu(req->max_fragmented_size)); 568 if (le16_to_cpu(req->min_version) > 0x0100 || 569 le16_to_cpu(req->max_version) < 0x0100) 570 return -EOPNOTSUPP; 571 if (le16_to_cpu(req->credits_requested) <= 0 || 572 le32_to_cpu(req->max_receive_size) <= 128 || 573 le32_to_cpu(req->max_fragmented_size) <= 574 128 * 1024) 575 return -ECONNABORTED; 576 577 return 0; 578 } 579 case SMBDIRECT_EXPECT_NEGOTIATE_REP: 580 /* client only */ 581 break; 582 } 583 584 /* This is an internal error */ 585 return -EINVAL; 586 } 587 588 static void recv_done(struct ib_cq *cq, struct ib_wc *wc) 589 { 590 struct smbdirect_recv_io *recvmsg; 591 struct smbdirect_socket *sc; 592 struct smbdirect_socket_parameters *sp; 593 594 recvmsg = container_of(wc->wr_cqe, struct smbdirect_recv_io, cqe); 595 sc = recvmsg->socket; 596 sp = &sc->parameters; 597 598 if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_RECV) { 599 put_recvmsg(sc, recvmsg); 600 if (wc->status != IB_WC_WR_FLUSH_ERR) { 601 pr_err("Recv error. status='%s (%d)' opcode=%d\n", 602 ib_wc_status_msg(wc->status), wc->status, 603 wc->opcode); 604 smb_direct_disconnect_rdma_connection(sc); 605 } 606 return; 607 } 608 609 ksmbd_debug(RDMA, "Recv completed. status='%s (%d)', opcode=%d\n", 610 ib_wc_status_msg(wc->status), wc->status, 611 wc->opcode); 612 613 ib_dma_sync_single_for_cpu(wc->qp->device, recvmsg->sge.addr, 614 recvmsg->sge.length, DMA_FROM_DEVICE); 615 616 /* 617 * Reset timer to the keepalive interval in 618 * order to trigger our next keepalive message. 619 */ 620 sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE; 621 mod_delayed_work(sc->workqueue, &sc->idle.timer_work, 622 msecs_to_jiffies(sp->keepalive_interval_msec)); 623 624 switch (sc->recv_io.expected) { 625 case SMBDIRECT_EXPECT_NEGOTIATE_REQ: 626 if (wc->byte_len < sizeof(struct smbdirect_negotiate_req)) { 627 put_recvmsg(sc, recvmsg); 628 smb_direct_disconnect_rdma_connection(sc); 629 return; 630 } 631 sc->recv_io.reassembly.full_packet_received = true; 632 WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_NEGOTIATE_NEEDED); 633 sc->status = SMBDIRECT_SOCKET_NEGOTIATE_RUNNING; 634 enqueue_reassembly(sc, recvmsg, 0); 635 wake_up(&sc->status_wait); 636 return; 637 case SMBDIRECT_EXPECT_DATA_TRANSFER: { 638 struct smbdirect_data_transfer *data_transfer = 639 (struct smbdirect_data_transfer *)recvmsg->packet; 640 u32 remaining_data_length, data_offset, data_length; 641 u16 old_recv_credit_target; 642 643 if (wc->byte_len < 644 offsetof(struct smbdirect_data_transfer, padding)) { 645 put_recvmsg(sc, recvmsg); 646 smb_direct_disconnect_rdma_connection(sc); 647 return; 648 } 649 650 remaining_data_length = le32_to_cpu(data_transfer->remaining_data_length); 651 data_length = le32_to_cpu(data_transfer->data_length); 652 data_offset = le32_to_cpu(data_transfer->data_offset); 653 if (wc->byte_len < data_offset || 654 wc->byte_len < (u64)data_offset + data_length) { 655 put_recvmsg(sc, recvmsg); 656 smb_direct_disconnect_rdma_connection(sc); 657 return; 658 } 659 if (remaining_data_length > sp->max_fragmented_recv_size || 660 data_length > sp->max_fragmented_recv_size || 661 (u64)remaining_data_length + (u64)data_length > 662 (u64)sp->max_fragmented_recv_size) { 663 put_recvmsg(sc, recvmsg); 664 smb_direct_disconnect_rdma_connection(sc); 665 return; 666 } 667 668 if (data_length) { 669 if (sc->recv_io.reassembly.full_packet_received) 670 recvmsg->first_segment = true; 671 672 if (le32_to_cpu(data_transfer->remaining_data_length)) 673 sc->recv_io.reassembly.full_packet_received = false; 674 else 675 sc->recv_io.reassembly.full_packet_received = true; 676 } 677 678 atomic_dec(&sc->recv_io.posted.count); 679 atomic_dec(&sc->recv_io.credits.count); 680 681 old_recv_credit_target = sc->recv_io.credits.target; 682 sc->recv_io.credits.target = 683 le16_to_cpu(data_transfer->credits_requested); 684 sc->recv_io.credits.target = 685 min_t(u16, sc->recv_io.credits.target, sp->recv_credit_max); 686 sc->recv_io.credits.target = 687 max_t(u16, sc->recv_io.credits.target, 1); 688 atomic_add(le16_to_cpu(data_transfer->credits_granted), 689 &sc->send_io.credits.count); 690 691 if (le16_to_cpu(data_transfer->flags) & 692 SMBDIRECT_FLAG_RESPONSE_REQUESTED) 693 queue_work(sc->workqueue, &sc->idle.immediate_work); 694 695 if (atomic_read(&sc->send_io.credits.count) > 0) 696 wake_up(&sc->send_io.credits.wait_queue); 697 698 if (data_length) { 699 if (sc->recv_io.credits.target > old_recv_credit_target) 700 queue_work(sc->workqueue, &sc->recv_io.posted.refill_work); 701 702 enqueue_reassembly(sc, recvmsg, (int)data_length); 703 wake_up(&sc->recv_io.reassembly.wait_queue); 704 } else 705 put_recvmsg(sc, recvmsg); 706 707 return; 708 } 709 case SMBDIRECT_EXPECT_NEGOTIATE_REP: 710 /* client only */ 711 break; 712 } 713 714 /* 715 * This is an internal error! 716 */ 717 WARN_ON_ONCE(sc->recv_io.expected != SMBDIRECT_EXPECT_DATA_TRANSFER); 718 put_recvmsg(sc, recvmsg); 719 smb_direct_disconnect_rdma_connection(sc); 720 } 721 722 static int smb_direct_post_recv(struct smbdirect_socket *sc, 723 struct smbdirect_recv_io *recvmsg) 724 { 725 struct smbdirect_socket_parameters *sp = &sc->parameters; 726 struct ib_recv_wr wr; 727 int ret; 728 729 recvmsg->sge.addr = ib_dma_map_single(sc->ib.dev, 730 recvmsg->packet, 731 sp->max_recv_size, 732 DMA_FROM_DEVICE); 733 ret = ib_dma_mapping_error(sc->ib.dev, recvmsg->sge.addr); 734 if (ret) 735 return ret; 736 recvmsg->sge.length = sp->max_recv_size; 737 recvmsg->sge.lkey = sc->ib.pd->local_dma_lkey; 738 recvmsg->cqe.done = recv_done; 739 740 wr.wr_cqe = &recvmsg->cqe; 741 wr.next = NULL; 742 wr.sg_list = &recvmsg->sge; 743 wr.num_sge = 1; 744 745 ret = ib_post_recv(sc->ib.qp, &wr, NULL); 746 if (ret) { 747 pr_err("Can't post recv: %d\n", ret); 748 ib_dma_unmap_single(sc->ib.dev, 749 recvmsg->sge.addr, recvmsg->sge.length, 750 DMA_FROM_DEVICE); 751 recvmsg->sge.length = 0; 752 smb_direct_disconnect_rdma_connection(sc); 753 return ret; 754 } 755 return ret; 756 } 757 758 static int smb_direct_read(struct ksmbd_transport *t, char *buf, 759 unsigned int size, int unused) 760 { 761 struct smbdirect_recv_io *recvmsg; 762 struct smbdirect_data_transfer *data_transfer; 763 int to_copy, to_read, data_read, offset; 764 u32 data_length, remaining_data_length, data_offset; 765 int rc; 766 struct smb_direct_transport *st = SMBD_TRANS(t); 767 struct smbdirect_socket *sc = &st->socket; 768 769 again: 770 if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { 771 pr_err("disconnected\n"); 772 return -ENOTCONN; 773 } 774 775 /* 776 * No need to hold the reassembly queue lock all the time as we are 777 * the only one reading from the front of the queue. The transport 778 * may add more entries to the back of the queue at the same time 779 */ 780 if (sc->recv_io.reassembly.data_length >= size) { 781 int queue_length; 782 int queue_removed = 0; 783 unsigned long flags; 784 785 /* 786 * Need to make sure reassembly_data_length is read before 787 * reading reassembly_queue_length and calling 788 * get_first_reassembly. This call is lock free 789 * as we never read at the end of the queue which are being 790 * updated in SOFTIRQ as more data is received 791 */ 792 virt_rmb(); 793 queue_length = sc->recv_io.reassembly.queue_length; 794 data_read = 0; 795 to_read = size; 796 offset = sc->recv_io.reassembly.first_entry_offset; 797 while (data_read < size) { 798 recvmsg = get_first_reassembly(sc); 799 data_transfer = smbdirect_recv_io_payload(recvmsg); 800 data_length = le32_to_cpu(data_transfer->data_length); 801 remaining_data_length = 802 le32_to_cpu(data_transfer->remaining_data_length); 803 data_offset = le32_to_cpu(data_transfer->data_offset); 804 805 /* 806 * The upper layer expects RFC1002 length at the 807 * beginning of the payload. Return it to indicate 808 * the total length of the packet. This minimize the 809 * change to upper layer packet processing logic. This 810 * will be eventually remove when an intermediate 811 * transport layer is added 812 */ 813 if (recvmsg->first_segment && size == 4) { 814 unsigned int rfc1002_len = 815 data_length + remaining_data_length; 816 *((__be32 *)buf) = cpu_to_be32(rfc1002_len); 817 data_read = 4; 818 recvmsg->first_segment = false; 819 ksmbd_debug(RDMA, 820 "returning rfc1002 length %d\n", 821 rfc1002_len); 822 goto read_rfc1002_done; 823 } 824 825 to_copy = min_t(int, data_length - offset, to_read); 826 memcpy(buf + data_read, (char *)data_transfer + data_offset + offset, 827 to_copy); 828 829 /* move on to the next buffer? */ 830 if (to_copy == data_length - offset) { 831 queue_length--; 832 /* 833 * No need to lock if we are not at the 834 * end of the queue 835 */ 836 if (queue_length) { 837 list_del(&recvmsg->list); 838 } else { 839 spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); 840 list_del(&recvmsg->list); 841 spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); 842 } 843 queue_removed++; 844 put_recvmsg(sc, recvmsg); 845 offset = 0; 846 } else { 847 offset += to_copy; 848 } 849 850 to_read -= to_copy; 851 data_read += to_copy; 852 } 853 854 spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); 855 sc->recv_io.reassembly.data_length -= data_read; 856 sc->recv_io.reassembly.queue_length -= queue_removed; 857 spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); 858 859 sc->recv_io.reassembly.first_entry_offset = offset; 860 ksmbd_debug(RDMA, 861 "returning to thread data_read=%d reassembly_data_length=%d first_entry_offset=%d\n", 862 data_read, sc->recv_io.reassembly.data_length, 863 sc->recv_io.reassembly.first_entry_offset); 864 read_rfc1002_done: 865 return data_read; 866 } 867 868 ksmbd_debug(RDMA, "wait_event on more data\n"); 869 rc = wait_event_interruptible(sc->recv_io.reassembly.wait_queue, 870 sc->recv_io.reassembly.data_length >= size || 871 sc->status != SMBDIRECT_SOCKET_CONNECTED); 872 if (rc) 873 return -EINTR; 874 875 goto again; 876 } 877 878 static void smb_direct_post_recv_credits(struct work_struct *work) 879 { 880 struct smbdirect_socket *sc = 881 container_of(work, struct smbdirect_socket, recv_io.posted.refill_work); 882 struct smbdirect_recv_io *recvmsg; 883 int credits = 0; 884 int ret; 885 886 if (atomic_read(&sc->recv_io.credits.count) < sc->recv_io.credits.target) { 887 while (true) { 888 recvmsg = get_free_recvmsg(sc); 889 if (!recvmsg) 890 break; 891 892 recvmsg->first_segment = false; 893 894 ret = smb_direct_post_recv(sc, recvmsg); 895 if (ret) { 896 pr_err("Can't post recv: %d\n", ret); 897 put_recvmsg(sc, recvmsg); 898 break; 899 } 900 credits++; 901 902 atomic_inc(&sc->recv_io.posted.count); 903 } 904 } 905 906 if (credits) 907 queue_work(sc->workqueue, &sc->idle.immediate_work); 908 } 909 910 static void send_done(struct ib_cq *cq, struct ib_wc *wc) 911 { 912 struct smbdirect_send_io *sendmsg, *sibling; 913 struct smbdirect_socket *sc; 914 struct list_head *pos, *prev, *end; 915 916 sendmsg = container_of(wc->wr_cqe, struct smbdirect_send_io, cqe); 917 sc = sendmsg->socket; 918 919 ksmbd_debug(RDMA, "Send completed. status='%s (%d)', opcode=%d\n", 920 ib_wc_status_msg(wc->status), wc->status, 921 wc->opcode); 922 923 if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_SEND) { 924 pr_err("Send error. status='%s (%d)', opcode=%d\n", 925 ib_wc_status_msg(wc->status), wc->status, 926 wc->opcode); 927 smb_direct_disconnect_rdma_connection(sc); 928 } 929 930 if (atomic_dec_and_test(&sc->send_io.pending.count)) 931 wake_up(&sc->send_io.pending.zero_wait_queue); 932 933 /* iterate and free the list of messages in reverse. the list's head 934 * is invalid. 935 */ 936 for (pos = &sendmsg->sibling_list, prev = pos->prev, end = sendmsg->sibling_list.next; 937 prev != end; pos = prev, prev = prev->prev) { 938 sibling = container_of(pos, struct smbdirect_send_io, sibling_list); 939 smb_direct_free_sendmsg(sc, sibling); 940 } 941 942 sibling = container_of(pos, struct smbdirect_send_io, sibling_list); 943 smb_direct_free_sendmsg(sc, sibling); 944 } 945 946 static int manage_credits_prior_sending(struct smbdirect_socket *sc) 947 { 948 int new_credits; 949 950 if (atomic_read(&sc->recv_io.credits.count) >= sc->recv_io.credits.target) 951 return 0; 952 953 new_credits = atomic_read(&sc->recv_io.posted.count); 954 if (new_credits == 0) 955 return 0; 956 957 new_credits -= atomic_read(&sc->recv_io.credits.count); 958 if (new_credits <= 0) 959 return 0; 960 961 atomic_add(new_credits, &sc->recv_io.credits.count); 962 return new_credits; 963 } 964 965 static int manage_keep_alive_before_sending(struct smbdirect_socket *sc) 966 { 967 struct smbdirect_socket_parameters *sp = &sc->parameters; 968 969 if (sc->idle.keepalive == SMBDIRECT_KEEPALIVE_PENDING) { 970 sc->idle.keepalive = SMBDIRECT_KEEPALIVE_SENT; 971 /* 972 * Now use the keepalive timeout (instead of keepalive interval) 973 * in order to wait for a response 974 */ 975 mod_delayed_work(sc->workqueue, &sc->idle.timer_work, 976 msecs_to_jiffies(sp->keepalive_timeout_msec)); 977 return 1; 978 } 979 return 0; 980 } 981 982 static int smb_direct_post_send(struct smbdirect_socket *sc, 983 struct ib_send_wr *wr) 984 { 985 int ret; 986 987 atomic_inc(&sc->send_io.pending.count); 988 ret = ib_post_send(sc->ib.qp, wr, NULL); 989 if (ret) { 990 pr_err("failed to post send: %d\n", ret); 991 if (atomic_dec_and_test(&sc->send_io.pending.count)) 992 wake_up(&sc->send_io.pending.zero_wait_queue); 993 smb_direct_disconnect_rdma_connection(sc); 994 } 995 return ret; 996 } 997 998 static void smb_direct_send_ctx_init(struct smbdirect_send_batch *send_ctx, 999 bool need_invalidate_rkey, 1000 unsigned int remote_key) 1001 { 1002 INIT_LIST_HEAD(&send_ctx->msg_list); 1003 send_ctx->wr_cnt = 0; 1004 send_ctx->need_invalidate_rkey = need_invalidate_rkey; 1005 send_ctx->remote_key = remote_key; 1006 } 1007 1008 static int smb_direct_flush_send_list(struct smbdirect_socket *sc, 1009 struct smbdirect_send_batch *send_ctx, 1010 bool is_last) 1011 { 1012 struct smbdirect_send_io *first, *last; 1013 int ret; 1014 1015 if (list_empty(&send_ctx->msg_list)) 1016 return 0; 1017 1018 first = list_first_entry(&send_ctx->msg_list, 1019 struct smbdirect_send_io, 1020 sibling_list); 1021 last = list_last_entry(&send_ctx->msg_list, 1022 struct smbdirect_send_io, 1023 sibling_list); 1024 1025 if (send_ctx->need_invalidate_rkey) { 1026 first->wr.opcode = IB_WR_SEND_WITH_INV; 1027 first->wr.ex.invalidate_rkey = send_ctx->remote_key; 1028 send_ctx->need_invalidate_rkey = false; 1029 send_ctx->remote_key = 0; 1030 } 1031 1032 last->wr.send_flags = IB_SEND_SIGNALED; 1033 last->wr.wr_cqe = &last->cqe; 1034 1035 ret = smb_direct_post_send(sc, &first->wr); 1036 if (!ret) { 1037 smb_direct_send_ctx_init(send_ctx, 1038 send_ctx->need_invalidate_rkey, 1039 send_ctx->remote_key); 1040 } else { 1041 atomic_add(send_ctx->wr_cnt, &sc->send_io.credits.count); 1042 wake_up(&sc->send_io.credits.wait_queue); 1043 list_for_each_entry_safe(first, last, &send_ctx->msg_list, 1044 sibling_list) { 1045 smb_direct_free_sendmsg(sc, first); 1046 } 1047 } 1048 return ret; 1049 } 1050 1051 static int wait_for_credits(struct smbdirect_socket *sc, 1052 wait_queue_head_t *waitq, atomic_t *total_credits, 1053 int needed) 1054 { 1055 int ret; 1056 1057 do { 1058 if (atomic_sub_return(needed, total_credits) >= 0) 1059 return 0; 1060 1061 atomic_add(needed, total_credits); 1062 ret = wait_event_interruptible(*waitq, 1063 atomic_read(total_credits) >= needed || 1064 sc->status != SMBDIRECT_SOCKET_CONNECTED); 1065 1066 if (sc->status != SMBDIRECT_SOCKET_CONNECTED) 1067 return -ENOTCONN; 1068 else if (ret < 0) 1069 return ret; 1070 } while (true); 1071 } 1072 1073 static int wait_for_send_credits(struct smbdirect_socket *sc, 1074 struct smbdirect_send_batch *send_ctx) 1075 { 1076 int ret; 1077 1078 if (send_ctx && 1079 (send_ctx->wr_cnt >= 16 || atomic_read(&sc->send_io.credits.count) <= 1)) { 1080 ret = smb_direct_flush_send_list(sc, send_ctx, false); 1081 if (ret) 1082 return ret; 1083 } 1084 1085 return wait_for_credits(sc, &sc->send_io.credits.wait_queue, &sc->send_io.credits.count, 1); 1086 } 1087 1088 static int wait_for_rw_credits(struct smbdirect_socket *sc, int credits) 1089 { 1090 return wait_for_credits(sc, 1091 &sc->rw_io.credits.wait_queue, 1092 &sc->rw_io.credits.count, 1093 credits); 1094 } 1095 1096 static int calc_rw_credits(struct smbdirect_socket *sc, 1097 char *buf, unsigned int len) 1098 { 1099 return DIV_ROUND_UP(get_buf_page_count(buf, len), 1100 sc->rw_io.credits.num_pages); 1101 } 1102 1103 static int smb_direct_create_header(struct smbdirect_socket *sc, 1104 int size, int remaining_data_length, 1105 struct smbdirect_send_io **sendmsg_out) 1106 { 1107 struct smbdirect_socket_parameters *sp = &sc->parameters; 1108 struct smbdirect_send_io *sendmsg; 1109 struct smbdirect_data_transfer *packet; 1110 int header_length; 1111 int ret; 1112 1113 sendmsg = smb_direct_alloc_sendmsg(sc); 1114 if (IS_ERR(sendmsg)) 1115 return PTR_ERR(sendmsg); 1116 1117 /* Fill in the packet header */ 1118 packet = (struct smbdirect_data_transfer *)sendmsg->packet; 1119 packet->credits_requested = cpu_to_le16(sp->send_credit_target); 1120 packet->credits_granted = cpu_to_le16(manage_credits_prior_sending(sc)); 1121 1122 packet->flags = 0; 1123 if (manage_keep_alive_before_sending(sc)) 1124 packet->flags |= cpu_to_le16(SMBDIRECT_FLAG_RESPONSE_REQUESTED); 1125 1126 packet->reserved = 0; 1127 if (!size) 1128 packet->data_offset = 0; 1129 else 1130 packet->data_offset = cpu_to_le32(24); 1131 packet->data_length = cpu_to_le32(size); 1132 packet->remaining_data_length = cpu_to_le32(remaining_data_length); 1133 packet->padding = 0; 1134 1135 ksmbd_debug(RDMA, 1136 "credits_requested=%d credits_granted=%d data_offset=%d data_length=%d remaining_data_length=%d\n", 1137 le16_to_cpu(packet->credits_requested), 1138 le16_to_cpu(packet->credits_granted), 1139 le32_to_cpu(packet->data_offset), 1140 le32_to_cpu(packet->data_length), 1141 le32_to_cpu(packet->remaining_data_length)); 1142 1143 /* Map the packet to DMA */ 1144 header_length = sizeof(struct smbdirect_data_transfer); 1145 /* If this is a packet without payload, don't send padding */ 1146 if (!size) 1147 header_length = 1148 offsetof(struct smbdirect_data_transfer, padding); 1149 1150 sendmsg->sge[0].addr = ib_dma_map_single(sc->ib.dev, 1151 (void *)packet, 1152 header_length, 1153 DMA_TO_DEVICE); 1154 ret = ib_dma_mapping_error(sc->ib.dev, sendmsg->sge[0].addr); 1155 if (ret) { 1156 smb_direct_free_sendmsg(sc, sendmsg); 1157 return ret; 1158 } 1159 1160 sendmsg->num_sge = 1; 1161 sendmsg->sge[0].length = header_length; 1162 sendmsg->sge[0].lkey = sc->ib.pd->local_dma_lkey; 1163 1164 *sendmsg_out = sendmsg; 1165 return 0; 1166 } 1167 1168 static int get_sg_list(void *buf, int size, struct scatterlist *sg_list, int nentries) 1169 { 1170 bool high = is_vmalloc_addr(buf); 1171 struct page *page; 1172 int offset, len; 1173 int i = 0; 1174 1175 if (size <= 0 || nentries < get_buf_page_count(buf, size)) 1176 return -EINVAL; 1177 1178 offset = offset_in_page(buf); 1179 buf -= offset; 1180 while (size > 0) { 1181 len = min_t(int, PAGE_SIZE - offset, size); 1182 if (high) 1183 page = vmalloc_to_page(buf); 1184 else 1185 page = kmap_to_page(buf); 1186 1187 if (!sg_list) 1188 return -EINVAL; 1189 sg_set_page(sg_list, page, len, offset); 1190 sg_list = sg_next(sg_list); 1191 1192 buf += PAGE_SIZE; 1193 size -= len; 1194 offset = 0; 1195 i++; 1196 } 1197 return i; 1198 } 1199 1200 static int get_mapped_sg_list(struct ib_device *device, void *buf, int size, 1201 struct scatterlist *sg_list, int nentries, 1202 enum dma_data_direction dir) 1203 { 1204 int npages; 1205 1206 npages = get_sg_list(buf, size, sg_list, nentries); 1207 if (npages < 0) 1208 return -EINVAL; 1209 return ib_dma_map_sg(device, sg_list, npages, dir); 1210 } 1211 1212 static int post_sendmsg(struct smbdirect_socket *sc, 1213 struct smbdirect_send_batch *send_ctx, 1214 struct smbdirect_send_io *msg) 1215 { 1216 int i; 1217 1218 for (i = 0; i < msg->num_sge; i++) 1219 ib_dma_sync_single_for_device(sc->ib.dev, 1220 msg->sge[i].addr, msg->sge[i].length, 1221 DMA_TO_DEVICE); 1222 1223 msg->cqe.done = send_done; 1224 msg->wr.opcode = IB_WR_SEND; 1225 msg->wr.sg_list = &msg->sge[0]; 1226 msg->wr.num_sge = msg->num_sge; 1227 msg->wr.next = NULL; 1228 1229 if (send_ctx) { 1230 msg->wr.wr_cqe = NULL; 1231 msg->wr.send_flags = 0; 1232 if (!list_empty(&send_ctx->msg_list)) { 1233 struct smbdirect_send_io *last; 1234 1235 last = list_last_entry(&send_ctx->msg_list, 1236 struct smbdirect_send_io, 1237 sibling_list); 1238 last->wr.next = &msg->wr; 1239 } 1240 list_add_tail(&msg->sibling_list, &send_ctx->msg_list); 1241 send_ctx->wr_cnt++; 1242 return 0; 1243 } 1244 1245 msg->wr.wr_cqe = &msg->cqe; 1246 msg->wr.send_flags = IB_SEND_SIGNALED; 1247 return smb_direct_post_send(sc, &msg->wr); 1248 } 1249 1250 static int smb_direct_post_send_data(struct smbdirect_socket *sc, 1251 struct smbdirect_send_batch *send_ctx, 1252 struct kvec *iov, int niov, 1253 int remaining_data_length) 1254 { 1255 int i, j, ret; 1256 struct smbdirect_send_io *msg; 1257 int data_length; 1258 struct scatterlist sg[SMBDIRECT_SEND_IO_MAX_SGE - 1]; 1259 1260 ret = wait_for_send_credits(sc, send_ctx); 1261 if (ret) 1262 return ret; 1263 1264 data_length = 0; 1265 for (i = 0; i < niov; i++) 1266 data_length += iov[i].iov_len; 1267 1268 ret = smb_direct_create_header(sc, data_length, remaining_data_length, 1269 &msg); 1270 if (ret) { 1271 atomic_inc(&sc->send_io.credits.count); 1272 return ret; 1273 } 1274 1275 for (i = 0; i < niov; i++) { 1276 struct ib_sge *sge; 1277 int sg_cnt; 1278 1279 sg_init_table(sg, SMBDIRECT_SEND_IO_MAX_SGE - 1); 1280 sg_cnt = get_mapped_sg_list(sc->ib.dev, 1281 iov[i].iov_base, iov[i].iov_len, 1282 sg, SMBDIRECT_SEND_IO_MAX_SGE - 1, 1283 DMA_TO_DEVICE); 1284 if (sg_cnt <= 0) { 1285 pr_err("failed to map buffer\n"); 1286 ret = -ENOMEM; 1287 goto err; 1288 } else if (sg_cnt + msg->num_sge > SMBDIRECT_SEND_IO_MAX_SGE) { 1289 pr_err("buffer not fitted into sges\n"); 1290 ret = -E2BIG; 1291 ib_dma_unmap_sg(sc->ib.dev, sg, sg_cnt, 1292 DMA_TO_DEVICE); 1293 goto err; 1294 } 1295 1296 for (j = 0; j < sg_cnt; j++) { 1297 sge = &msg->sge[msg->num_sge]; 1298 sge->addr = sg_dma_address(&sg[j]); 1299 sge->length = sg_dma_len(&sg[j]); 1300 sge->lkey = sc->ib.pd->local_dma_lkey; 1301 msg->num_sge++; 1302 } 1303 } 1304 1305 ret = post_sendmsg(sc, send_ctx, msg); 1306 if (ret) 1307 goto err; 1308 return 0; 1309 err: 1310 smb_direct_free_sendmsg(sc, msg); 1311 atomic_inc(&sc->send_io.credits.count); 1312 return ret; 1313 } 1314 1315 static int smb_direct_writev(struct ksmbd_transport *t, 1316 struct kvec *iov, int niovs, int buflen, 1317 bool need_invalidate, unsigned int remote_key) 1318 { 1319 struct smb_direct_transport *st = SMBD_TRANS(t); 1320 struct smbdirect_socket *sc = &st->socket; 1321 struct smbdirect_socket_parameters *sp = &sc->parameters; 1322 size_t remaining_data_length; 1323 size_t iov_idx; 1324 size_t iov_ofs; 1325 size_t max_iov_size = sp->max_send_size - 1326 sizeof(struct smbdirect_data_transfer); 1327 int ret; 1328 struct smbdirect_send_batch send_ctx; 1329 int error = 0; 1330 1331 if (sc->status != SMBDIRECT_SOCKET_CONNECTED) 1332 return -ENOTCONN; 1333 1334 //FIXME: skip RFC1002 header.. 1335 if (WARN_ON_ONCE(niovs <= 1 || iov[0].iov_len != 4)) 1336 return -EINVAL; 1337 buflen -= 4; 1338 iov_idx = 1; 1339 iov_ofs = 0; 1340 1341 remaining_data_length = buflen; 1342 ksmbd_debug(RDMA, "Sending smb (RDMA): smb_len=%u\n", buflen); 1343 1344 smb_direct_send_ctx_init(&send_ctx, need_invalidate, remote_key); 1345 while (remaining_data_length) { 1346 struct kvec vecs[SMBDIRECT_SEND_IO_MAX_SGE - 1]; /* minus smbdirect hdr */ 1347 size_t possible_bytes = max_iov_size; 1348 size_t possible_vecs; 1349 size_t bytes = 0; 1350 size_t nvecs = 0; 1351 1352 /* 1353 * For the last message remaining_data_length should be 1354 * have been 0 already! 1355 */ 1356 if (WARN_ON_ONCE(iov_idx >= niovs)) { 1357 error = -EINVAL; 1358 goto done; 1359 } 1360 1361 /* 1362 * We have 2 factors which limit the arguments we pass 1363 * to smb_direct_post_send_data(): 1364 * 1365 * 1. The number of supported sges for the send, 1366 * while one is reserved for the smbdirect header. 1367 * And we currently need one SGE per page. 1368 * 2. The number of negotiated payload bytes per send. 1369 */ 1370 possible_vecs = min_t(size_t, ARRAY_SIZE(vecs), niovs - iov_idx); 1371 1372 while (iov_idx < niovs && possible_vecs && possible_bytes) { 1373 struct kvec *v = &vecs[nvecs]; 1374 int page_count; 1375 1376 v->iov_base = ((u8 *)iov[iov_idx].iov_base) + iov_ofs; 1377 v->iov_len = min_t(size_t, 1378 iov[iov_idx].iov_len - iov_ofs, 1379 possible_bytes); 1380 page_count = get_buf_page_count(v->iov_base, v->iov_len); 1381 if (page_count > possible_vecs) { 1382 /* 1383 * If the number of pages in the buffer 1384 * is to much (because we currently require 1385 * one SGE per page), we need to limit the 1386 * length. 1387 * 1388 * We know possible_vecs is at least 1, 1389 * so we always keep the first page. 1390 * 1391 * We need to calculate the number extra 1392 * pages (epages) we can also keep. 1393 * 1394 * We calculate the number of bytes in the 1395 * first page (fplen), this should never be 1396 * larger than v->iov_len because page_count is 1397 * at least 2, but adding a limitation feels 1398 * better. 1399 * 1400 * Then we calculate the number of bytes (elen) 1401 * we can keep for the extra pages. 1402 */ 1403 size_t epages = possible_vecs - 1; 1404 size_t fpofs = offset_in_page(v->iov_base); 1405 size_t fplen = min_t(size_t, PAGE_SIZE - fpofs, v->iov_len); 1406 size_t elen = min_t(size_t, v->iov_len - fplen, epages*PAGE_SIZE); 1407 1408 v->iov_len = fplen + elen; 1409 page_count = get_buf_page_count(v->iov_base, v->iov_len); 1410 if (WARN_ON_ONCE(page_count > possible_vecs)) { 1411 /* 1412 * Something went wrong in the above 1413 * logic... 1414 */ 1415 error = -EINVAL; 1416 goto done; 1417 } 1418 } 1419 possible_vecs -= page_count; 1420 nvecs += 1; 1421 possible_bytes -= v->iov_len; 1422 bytes += v->iov_len; 1423 1424 iov_ofs += v->iov_len; 1425 if (iov_ofs >= iov[iov_idx].iov_len) { 1426 iov_idx += 1; 1427 iov_ofs = 0; 1428 } 1429 } 1430 1431 remaining_data_length -= bytes; 1432 1433 ret = smb_direct_post_send_data(sc, &send_ctx, 1434 vecs, nvecs, 1435 remaining_data_length); 1436 if (unlikely(ret)) { 1437 error = ret; 1438 goto done; 1439 } 1440 } 1441 1442 done: 1443 ret = smb_direct_flush_send_list(sc, &send_ctx, true); 1444 if (unlikely(!ret && error)) 1445 ret = error; 1446 1447 /* 1448 * As an optimization, we don't wait for individual I/O to finish 1449 * before sending the next one. 1450 * Send them all and wait for pending send count to get to 0 1451 * that means all the I/Os have been out and we are good to return 1452 */ 1453 1454 wait_event(sc->send_io.pending.zero_wait_queue, 1455 atomic_read(&sc->send_io.pending.count) == 0 || 1456 sc->status != SMBDIRECT_SOCKET_CONNECTED); 1457 if (sc->status != SMBDIRECT_SOCKET_CONNECTED && ret == 0) 1458 ret = -ENOTCONN; 1459 1460 return ret; 1461 } 1462 1463 static void smb_direct_free_rdma_rw_msg(struct smb_direct_transport *t, 1464 struct smbdirect_rw_io *msg, 1465 enum dma_data_direction dir) 1466 { 1467 struct smbdirect_socket *sc = &t->socket; 1468 1469 rdma_rw_ctx_destroy(&msg->rdma_ctx, sc->ib.qp, sc->ib.qp->port, 1470 msg->sgt.sgl, msg->sgt.nents, dir); 1471 sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE); 1472 kfree(msg); 1473 } 1474 1475 static void read_write_done(struct ib_cq *cq, struct ib_wc *wc, 1476 enum dma_data_direction dir) 1477 { 1478 struct smbdirect_rw_io *msg = 1479 container_of(wc->wr_cqe, struct smbdirect_rw_io, cqe); 1480 struct smbdirect_socket *sc = msg->socket; 1481 1482 if (wc->status != IB_WC_SUCCESS) { 1483 msg->error = -EIO; 1484 pr_err("read/write error. opcode = %d, status = %s(%d)\n", 1485 wc->opcode, ib_wc_status_msg(wc->status), wc->status); 1486 if (wc->status != IB_WC_WR_FLUSH_ERR) 1487 smb_direct_disconnect_rdma_connection(sc); 1488 } 1489 1490 complete(msg->completion); 1491 } 1492 1493 static void read_done(struct ib_cq *cq, struct ib_wc *wc) 1494 { 1495 read_write_done(cq, wc, DMA_FROM_DEVICE); 1496 } 1497 1498 static void write_done(struct ib_cq *cq, struct ib_wc *wc) 1499 { 1500 read_write_done(cq, wc, DMA_TO_DEVICE); 1501 } 1502 1503 static int smb_direct_rdma_xmit(struct smb_direct_transport *t, 1504 void *buf, int buf_len, 1505 struct smbdirect_buffer_descriptor_v1 *desc, 1506 unsigned int desc_len, 1507 bool is_read) 1508 { 1509 struct smbdirect_socket *sc = &t->socket; 1510 struct smbdirect_socket_parameters *sp = &sc->parameters; 1511 struct smbdirect_rw_io *msg, *next_msg; 1512 int i, ret; 1513 DECLARE_COMPLETION_ONSTACK(completion); 1514 struct ib_send_wr *first_wr; 1515 LIST_HEAD(msg_list); 1516 char *desc_buf; 1517 int credits_needed; 1518 unsigned int desc_buf_len, desc_num = 0; 1519 1520 if (sc->status != SMBDIRECT_SOCKET_CONNECTED) 1521 return -ENOTCONN; 1522 1523 if (buf_len > sp->max_read_write_size) 1524 return -EINVAL; 1525 1526 /* calculate needed credits */ 1527 credits_needed = 0; 1528 desc_buf = buf; 1529 for (i = 0; i < desc_len / sizeof(*desc); i++) { 1530 if (!buf_len) 1531 break; 1532 1533 desc_buf_len = le32_to_cpu(desc[i].length); 1534 if (!desc_buf_len) 1535 return -EINVAL; 1536 1537 if (desc_buf_len > buf_len) { 1538 desc_buf_len = buf_len; 1539 desc[i].length = cpu_to_le32(desc_buf_len); 1540 buf_len = 0; 1541 } 1542 1543 credits_needed += calc_rw_credits(sc, desc_buf, desc_buf_len); 1544 desc_buf += desc_buf_len; 1545 buf_len -= desc_buf_len; 1546 desc_num++; 1547 } 1548 1549 ksmbd_debug(RDMA, "RDMA %s, len %#x, needed credits %#x\n", 1550 str_read_write(is_read), buf_len, credits_needed); 1551 1552 ret = wait_for_rw_credits(sc, credits_needed); 1553 if (ret < 0) 1554 return ret; 1555 1556 /* build rdma_rw_ctx for each descriptor */ 1557 desc_buf = buf; 1558 for (i = 0; i < desc_num; i++) { 1559 msg = kzalloc(struct_size(msg, sg_list, SG_CHUNK_SIZE), 1560 KSMBD_DEFAULT_GFP); 1561 if (!msg) { 1562 ret = -ENOMEM; 1563 goto out; 1564 } 1565 1566 desc_buf_len = le32_to_cpu(desc[i].length); 1567 1568 msg->socket = sc; 1569 msg->cqe.done = is_read ? read_done : write_done; 1570 msg->completion = &completion; 1571 1572 msg->sgt.sgl = &msg->sg_list[0]; 1573 ret = sg_alloc_table_chained(&msg->sgt, 1574 get_buf_page_count(desc_buf, desc_buf_len), 1575 msg->sg_list, SG_CHUNK_SIZE); 1576 if (ret) { 1577 kfree(msg); 1578 ret = -ENOMEM; 1579 goto out; 1580 } 1581 1582 ret = get_sg_list(desc_buf, desc_buf_len, 1583 msg->sgt.sgl, msg->sgt.orig_nents); 1584 if (ret < 0) { 1585 sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE); 1586 kfree(msg); 1587 goto out; 1588 } 1589 1590 ret = rdma_rw_ctx_init(&msg->rdma_ctx, sc->ib.qp, sc->ib.qp->port, 1591 msg->sgt.sgl, 1592 get_buf_page_count(desc_buf, desc_buf_len), 1593 0, 1594 le64_to_cpu(desc[i].offset), 1595 le32_to_cpu(desc[i].token), 1596 is_read ? DMA_FROM_DEVICE : DMA_TO_DEVICE); 1597 if (ret < 0) { 1598 pr_err("failed to init rdma_rw_ctx: %d\n", ret); 1599 sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE); 1600 kfree(msg); 1601 goto out; 1602 } 1603 1604 list_add_tail(&msg->list, &msg_list); 1605 desc_buf += desc_buf_len; 1606 } 1607 1608 /* concatenate work requests of rdma_rw_ctxs */ 1609 first_wr = NULL; 1610 list_for_each_entry_reverse(msg, &msg_list, list) { 1611 first_wr = rdma_rw_ctx_wrs(&msg->rdma_ctx, sc->ib.qp, sc->ib.qp->port, 1612 &msg->cqe, first_wr); 1613 } 1614 1615 ret = ib_post_send(sc->ib.qp, first_wr, NULL); 1616 if (ret) { 1617 pr_err("failed to post send wr for RDMA R/W: %d\n", ret); 1618 goto out; 1619 } 1620 1621 msg = list_last_entry(&msg_list, struct smbdirect_rw_io, list); 1622 wait_for_completion(&completion); 1623 ret = msg->error; 1624 out: 1625 list_for_each_entry_safe(msg, next_msg, &msg_list, list) { 1626 list_del(&msg->list); 1627 smb_direct_free_rdma_rw_msg(t, msg, 1628 is_read ? DMA_FROM_DEVICE : DMA_TO_DEVICE); 1629 } 1630 atomic_add(credits_needed, &sc->rw_io.credits.count); 1631 wake_up(&sc->rw_io.credits.wait_queue); 1632 return ret; 1633 } 1634 1635 static int smb_direct_rdma_write(struct ksmbd_transport *t, 1636 void *buf, unsigned int buflen, 1637 struct smbdirect_buffer_descriptor_v1 *desc, 1638 unsigned int desc_len) 1639 { 1640 return smb_direct_rdma_xmit(SMBD_TRANS(t), buf, buflen, 1641 desc, desc_len, false); 1642 } 1643 1644 static int smb_direct_rdma_read(struct ksmbd_transport *t, 1645 void *buf, unsigned int buflen, 1646 struct smbdirect_buffer_descriptor_v1 *desc, 1647 unsigned int desc_len) 1648 { 1649 return smb_direct_rdma_xmit(SMBD_TRANS(t), buf, buflen, 1650 desc, desc_len, true); 1651 } 1652 1653 static void smb_direct_disconnect(struct ksmbd_transport *t) 1654 { 1655 struct smb_direct_transport *st = SMBD_TRANS(t); 1656 struct smbdirect_socket *sc = &st->socket; 1657 1658 ksmbd_debug(RDMA, "Disconnecting cm_id=%p\n", sc->rdma.cm_id); 1659 1660 free_transport(st); 1661 } 1662 1663 static void smb_direct_shutdown(struct ksmbd_transport *t) 1664 { 1665 struct smb_direct_transport *st = SMBD_TRANS(t); 1666 struct smbdirect_socket *sc = &st->socket; 1667 1668 ksmbd_debug(RDMA, "smb-direct shutdown cm_id=%p\n", sc->rdma.cm_id); 1669 1670 smb_direct_disconnect_rdma_work(&sc->disconnect_work); 1671 } 1672 1673 static int smb_direct_cm_handler(struct rdma_cm_id *cm_id, 1674 struct rdma_cm_event *event) 1675 { 1676 struct smbdirect_socket *sc = cm_id->context; 1677 1678 ksmbd_debug(RDMA, "RDMA CM event. cm_id=%p event=%s (%d)\n", 1679 cm_id, rdma_event_msg(event->event), event->event); 1680 1681 switch (event->event) { 1682 case RDMA_CM_EVENT_ESTABLISHED: { 1683 WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING); 1684 sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED; 1685 wake_up(&sc->status_wait); 1686 break; 1687 } 1688 case RDMA_CM_EVENT_DEVICE_REMOVAL: 1689 case RDMA_CM_EVENT_DISCONNECTED: { 1690 ib_drain_qp(sc->ib.qp); 1691 1692 sc->status = SMBDIRECT_SOCKET_DISCONNECTED; 1693 smb_direct_disconnect_rdma_work(&sc->disconnect_work); 1694 break; 1695 } 1696 case RDMA_CM_EVENT_CONNECT_ERROR: { 1697 sc->status = SMBDIRECT_SOCKET_DISCONNECTED; 1698 smb_direct_disconnect_rdma_work(&sc->disconnect_work); 1699 break; 1700 } 1701 default: 1702 pr_err("Unexpected RDMA CM event. cm_id=%p, event=%s (%d)\n", 1703 cm_id, rdma_event_msg(event->event), 1704 event->event); 1705 break; 1706 } 1707 return 0; 1708 } 1709 1710 static void smb_direct_qpair_handler(struct ib_event *event, void *context) 1711 { 1712 struct smbdirect_socket *sc = context; 1713 1714 ksmbd_debug(RDMA, "Received QP event. cm_id=%p, event=%s (%d)\n", 1715 sc->rdma.cm_id, ib_event_msg(event->event), event->event); 1716 1717 switch (event->event) { 1718 case IB_EVENT_CQ_ERR: 1719 case IB_EVENT_QP_FATAL: 1720 smb_direct_disconnect_rdma_connection(sc); 1721 break; 1722 default: 1723 break; 1724 } 1725 } 1726 1727 static int smb_direct_send_negotiate_response(struct smbdirect_socket *sc, 1728 int failed) 1729 { 1730 struct smbdirect_socket_parameters *sp = &sc->parameters; 1731 struct smbdirect_send_io *sendmsg; 1732 struct smbdirect_negotiate_resp *resp; 1733 int ret; 1734 1735 sendmsg = smb_direct_alloc_sendmsg(sc); 1736 if (IS_ERR(sendmsg)) 1737 return -ENOMEM; 1738 1739 resp = (struct smbdirect_negotiate_resp *)sendmsg->packet; 1740 if (failed) { 1741 memset(resp, 0, sizeof(*resp)); 1742 resp->min_version = SMB_DIRECT_VERSION_LE; 1743 resp->max_version = SMB_DIRECT_VERSION_LE; 1744 resp->status = STATUS_NOT_SUPPORTED; 1745 1746 sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED; 1747 } else { 1748 resp->status = STATUS_SUCCESS; 1749 resp->min_version = SMB_DIRECT_VERSION_LE; 1750 resp->max_version = SMB_DIRECT_VERSION_LE; 1751 resp->negotiated_version = SMB_DIRECT_VERSION_LE; 1752 resp->reserved = 0; 1753 resp->credits_requested = 1754 cpu_to_le16(sp->send_credit_target); 1755 resp->credits_granted = cpu_to_le16(manage_credits_prior_sending(sc)); 1756 resp->max_readwrite_size = cpu_to_le32(sp->max_read_write_size); 1757 resp->preferred_send_size = cpu_to_le32(sp->max_send_size); 1758 resp->max_receive_size = cpu_to_le32(sp->max_recv_size); 1759 resp->max_fragmented_size = 1760 cpu_to_le32(sp->max_fragmented_recv_size); 1761 1762 sc->recv_io.expected = SMBDIRECT_EXPECT_DATA_TRANSFER; 1763 sc->status = SMBDIRECT_SOCKET_CONNECTED; 1764 } 1765 1766 sendmsg->sge[0].addr = ib_dma_map_single(sc->ib.dev, 1767 (void *)resp, sizeof(*resp), 1768 DMA_TO_DEVICE); 1769 ret = ib_dma_mapping_error(sc->ib.dev, sendmsg->sge[0].addr); 1770 if (ret) { 1771 smb_direct_free_sendmsg(sc, sendmsg); 1772 return ret; 1773 } 1774 1775 sendmsg->num_sge = 1; 1776 sendmsg->sge[0].length = sizeof(*resp); 1777 sendmsg->sge[0].lkey = sc->ib.pd->local_dma_lkey; 1778 1779 ret = post_sendmsg(sc, NULL, sendmsg); 1780 if (ret) { 1781 smb_direct_free_sendmsg(sc, sendmsg); 1782 return ret; 1783 } 1784 1785 wait_event(sc->send_io.pending.zero_wait_queue, 1786 atomic_read(&sc->send_io.pending.count) == 0 || 1787 sc->status != SMBDIRECT_SOCKET_CONNECTED); 1788 if (sc->status != SMBDIRECT_SOCKET_CONNECTED) 1789 return -ENOTCONN; 1790 1791 return 0; 1792 } 1793 1794 static int smb_direct_accept_client(struct smbdirect_socket *sc) 1795 { 1796 struct smbdirect_socket_parameters *sp = &sc->parameters; 1797 struct rdma_conn_param conn_param; 1798 __be32 ird_ord_hdr[2]; 1799 int ret; 1800 1801 /* 1802 * smb_direct_handle_connect_request() 1803 * already negotiated sp->initiator_depth 1804 * and sp->responder_resources 1805 */ 1806 memset(&conn_param, 0, sizeof(conn_param)); 1807 conn_param.initiator_depth = sp->initiator_depth; 1808 conn_param.responder_resources = sp->responder_resources; 1809 1810 if (sc->rdma.legacy_iwarp) { 1811 ird_ord_hdr[0] = cpu_to_be32(conn_param.responder_resources); 1812 ird_ord_hdr[1] = cpu_to_be32(conn_param.initiator_depth); 1813 conn_param.private_data = ird_ord_hdr; 1814 conn_param.private_data_len = sizeof(ird_ord_hdr); 1815 } else { 1816 conn_param.private_data = NULL; 1817 conn_param.private_data_len = 0; 1818 } 1819 conn_param.retry_count = SMB_DIRECT_CM_RETRY; 1820 conn_param.rnr_retry_count = SMB_DIRECT_CM_RNR_RETRY; 1821 conn_param.flow_control = 0; 1822 1823 /* 1824 * start with the negotiate timeout and SMBDIRECT_KEEPALIVE_PENDING 1825 * so that the timer will cause a disconnect. 1826 */ 1827 sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING; 1828 mod_delayed_work(sc->workqueue, &sc->idle.timer_work, 1829 msecs_to_jiffies(sp->negotiate_timeout_msec)); 1830 1831 WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED); 1832 sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING; 1833 ret = rdma_accept(sc->rdma.cm_id, &conn_param); 1834 if (ret) { 1835 pr_err("error at rdma_accept: %d\n", ret); 1836 return ret; 1837 } 1838 return 0; 1839 } 1840 1841 static int smb_direct_prepare_negotiation(struct smbdirect_socket *sc) 1842 { 1843 struct smbdirect_recv_io *recvmsg; 1844 int ret; 1845 1846 WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED); 1847 sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED; 1848 1849 sc->recv_io.expected = SMBDIRECT_EXPECT_NEGOTIATE_REQ; 1850 1851 recvmsg = get_free_recvmsg(sc); 1852 if (!recvmsg) 1853 return -ENOMEM; 1854 1855 ret = smb_direct_post_recv(sc, recvmsg); 1856 if (ret) { 1857 pr_err("Can't post recv: %d\n", ret); 1858 goto out_err; 1859 } 1860 1861 ret = smb_direct_accept_client(sc); 1862 if (ret) { 1863 pr_err("Can't accept client\n"); 1864 goto out_err; 1865 } 1866 1867 smb_direct_post_recv_credits(&sc->recv_io.posted.refill_work); 1868 return 0; 1869 out_err: 1870 put_recvmsg(sc, recvmsg); 1871 return ret; 1872 } 1873 1874 static unsigned int smb_direct_get_max_fr_pages(struct smbdirect_socket *sc) 1875 { 1876 return min_t(unsigned int, 1877 sc->ib.dev->attrs.max_fast_reg_page_list_len, 1878 256); 1879 } 1880 1881 static int smb_direct_init_params(struct smbdirect_socket *sc, 1882 struct ib_qp_cap *cap) 1883 { 1884 struct smbdirect_socket_parameters *sp = &sc->parameters; 1885 struct ib_device *device = sc->ib.dev; 1886 int max_send_sges, max_rw_wrs, max_send_wrs; 1887 unsigned int max_sge_per_wr, wrs_per_credit; 1888 1889 /* need 3 more sge. because a SMB_DIRECT header, SMB2 header, 1890 * SMB2 response could be mapped. 1891 */ 1892 max_send_sges = DIV_ROUND_UP(sp->max_send_size, PAGE_SIZE) + 3; 1893 if (max_send_sges > SMBDIRECT_SEND_IO_MAX_SGE) { 1894 pr_err("max_send_size %d is too large\n", sp->max_send_size); 1895 return -EINVAL; 1896 } 1897 1898 /* Calculate the number of work requests for RDMA R/W. 1899 * The maximum number of pages which can be registered 1900 * with one Memory region can be transferred with one 1901 * R/W credit. And at least 4 work requests for each credit 1902 * are needed for MR registration, RDMA R/W, local & remote 1903 * MR invalidation. 1904 */ 1905 sc->rw_io.credits.num_pages = smb_direct_get_max_fr_pages(sc); 1906 sc->rw_io.credits.max = DIV_ROUND_UP(sp->max_read_write_size, 1907 (sc->rw_io.credits.num_pages - 1) * 1908 PAGE_SIZE); 1909 1910 max_sge_per_wr = min_t(unsigned int, device->attrs.max_send_sge, 1911 device->attrs.max_sge_rd); 1912 max_sge_per_wr = max_t(unsigned int, max_sge_per_wr, 1913 max_send_sges); 1914 wrs_per_credit = max_t(unsigned int, 4, 1915 DIV_ROUND_UP(sc->rw_io.credits.num_pages, 1916 max_sge_per_wr) + 1); 1917 max_rw_wrs = sc->rw_io.credits.max * wrs_per_credit; 1918 1919 max_send_wrs = sp->send_credit_target + max_rw_wrs; 1920 if (max_send_wrs > device->attrs.max_cqe || 1921 max_send_wrs > device->attrs.max_qp_wr) { 1922 pr_err("consider lowering send_credit_target = %d\n", 1923 sp->send_credit_target); 1924 pr_err("Possible CQE overrun, device reporting max_cqe %d max_qp_wr %d\n", 1925 device->attrs.max_cqe, device->attrs.max_qp_wr); 1926 return -EINVAL; 1927 } 1928 1929 if (sp->recv_credit_max > device->attrs.max_cqe || 1930 sp->recv_credit_max > device->attrs.max_qp_wr) { 1931 pr_err("consider lowering receive_credit_max = %d\n", 1932 sp->recv_credit_max); 1933 pr_err("Possible CQE overrun, device reporting max_cpe %d max_qp_wr %d\n", 1934 device->attrs.max_cqe, device->attrs.max_qp_wr); 1935 return -EINVAL; 1936 } 1937 1938 if (device->attrs.max_send_sge < SMBDIRECT_SEND_IO_MAX_SGE) { 1939 pr_err("warning: device max_send_sge = %d too small\n", 1940 device->attrs.max_send_sge); 1941 return -EINVAL; 1942 } 1943 if (device->attrs.max_recv_sge < SMBDIRECT_RECV_IO_MAX_SGE) { 1944 pr_err("warning: device max_recv_sge = %d too small\n", 1945 device->attrs.max_recv_sge); 1946 return -EINVAL; 1947 } 1948 1949 sc->recv_io.credits.target = 1; 1950 1951 atomic_set(&sc->rw_io.credits.count, sc->rw_io.credits.max); 1952 1953 cap->max_send_wr = max_send_wrs; 1954 cap->max_recv_wr = sp->recv_credit_max; 1955 cap->max_send_sge = SMBDIRECT_SEND_IO_MAX_SGE; 1956 cap->max_recv_sge = SMBDIRECT_RECV_IO_MAX_SGE; 1957 cap->max_inline_data = 0; 1958 cap->max_rdma_ctxs = sc->rw_io.credits.max; 1959 return 0; 1960 } 1961 1962 static void smb_direct_destroy_pools(struct smbdirect_socket *sc) 1963 { 1964 struct smbdirect_recv_io *recvmsg; 1965 1966 while ((recvmsg = get_free_recvmsg(sc))) 1967 mempool_free(recvmsg, sc->recv_io.mem.pool); 1968 1969 mempool_destroy(sc->recv_io.mem.pool); 1970 sc->recv_io.mem.pool = NULL; 1971 1972 kmem_cache_destroy(sc->recv_io.mem.cache); 1973 sc->recv_io.mem.cache = NULL; 1974 1975 mempool_destroy(sc->send_io.mem.pool); 1976 sc->send_io.mem.pool = NULL; 1977 1978 kmem_cache_destroy(sc->send_io.mem.cache); 1979 sc->send_io.mem.cache = NULL; 1980 } 1981 1982 static int smb_direct_create_pools(struct smbdirect_socket *sc) 1983 { 1984 struct smbdirect_socket_parameters *sp = &sc->parameters; 1985 char name[80]; 1986 int i; 1987 struct smbdirect_recv_io *recvmsg; 1988 1989 snprintf(name, sizeof(name), "smbdirect_send_io_pool_%p", sc); 1990 sc->send_io.mem.cache = kmem_cache_create(name, 1991 sizeof(struct smbdirect_send_io) + 1992 sizeof(struct smbdirect_negotiate_resp), 1993 0, SLAB_HWCACHE_ALIGN, NULL); 1994 if (!sc->send_io.mem.cache) 1995 return -ENOMEM; 1996 1997 sc->send_io.mem.pool = mempool_create(sp->send_credit_target, 1998 mempool_alloc_slab, mempool_free_slab, 1999 sc->send_io.mem.cache); 2000 if (!sc->send_io.mem.pool) 2001 goto err; 2002 2003 snprintf(name, sizeof(name), "smbdirect_recv_io_pool_%p", sc); 2004 sc->recv_io.mem.cache = kmem_cache_create(name, 2005 sizeof(struct smbdirect_recv_io) + 2006 sp->max_recv_size, 2007 0, SLAB_HWCACHE_ALIGN, NULL); 2008 if (!sc->recv_io.mem.cache) 2009 goto err; 2010 2011 sc->recv_io.mem.pool = 2012 mempool_create(sp->recv_credit_max, mempool_alloc_slab, 2013 mempool_free_slab, sc->recv_io.mem.cache); 2014 if (!sc->recv_io.mem.pool) 2015 goto err; 2016 2017 for (i = 0; i < sp->recv_credit_max; i++) { 2018 recvmsg = mempool_alloc(sc->recv_io.mem.pool, KSMBD_DEFAULT_GFP); 2019 if (!recvmsg) 2020 goto err; 2021 recvmsg->socket = sc; 2022 recvmsg->sge.length = 0; 2023 list_add(&recvmsg->list, &sc->recv_io.free.list); 2024 } 2025 2026 return 0; 2027 err: 2028 smb_direct_destroy_pools(sc); 2029 return -ENOMEM; 2030 } 2031 2032 static int smb_direct_create_qpair(struct smbdirect_socket *sc, 2033 struct ib_qp_cap *cap) 2034 { 2035 struct smbdirect_socket_parameters *sp = &sc->parameters; 2036 int ret; 2037 struct ib_qp_init_attr qp_attr; 2038 int pages_per_rw; 2039 2040 sc->ib.pd = ib_alloc_pd(sc->ib.dev, 0); 2041 if (IS_ERR(sc->ib.pd)) { 2042 pr_err("Can't create RDMA PD\n"); 2043 ret = PTR_ERR(sc->ib.pd); 2044 sc->ib.pd = NULL; 2045 return ret; 2046 } 2047 2048 sc->ib.send_cq = ib_alloc_cq_any(sc->ib.dev, sc, 2049 sp->send_credit_target + 2050 cap->max_rdma_ctxs, 2051 IB_POLL_WORKQUEUE); 2052 if (IS_ERR(sc->ib.send_cq)) { 2053 pr_err("Can't create RDMA send CQ\n"); 2054 ret = PTR_ERR(sc->ib.send_cq); 2055 sc->ib.send_cq = NULL; 2056 goto err; 2057 } 2058 2059 sc->ib.recv_cq = ib_alloc_cq_any(sc->ib.dev, sc, 2060 sp->recv_credit_max, 2061 IB_POLL_WORKQUEUE); 2062 if (IS_ERR(sc->ib.recv_cq)) { 2063 pr_err("Can't create RDMA recv CQ\n"); 2064 ret = PTR_ERR(sc->ib.recv_cq); 2065 sc->ib.recv_cq = NULL; 2066 goto err; 2067 } 2068 2069 memset(&qp_attr, 0, sizeof(qp_attr)); 2070 qp_attr.event_handler = smb_direct_qpair_handler; 2071 qp_attr.qp_context = sc; 2072 qp_attr.cap = *cap; 2073 qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR; 2074 qp_attr.qp_type = IB_QPT_RC; 2075 qp_attr.send_cq = sc->ib.send_cq; 2076 qp_attr.recv_cq = sc->ib.recv_cq; 2077 qp_attr.port_num = ~0; 2078 2079 ret = rdma_create_qp(sc->rdma.cm_id, sc->ib.pd, &qp_attr); 2080 if (ret) { 2081 pr_err("Can't create RDMA QP: %d\n", ret); 2082 goto err; 2083 } 2084 2085 sc->ib.qp = sc->rdma.cm_id->qp; 2086 sc->rdma.cm_id->event_handler = smb_direct_cm_handler; 2087 2088 pages_per_rw = DIV_ROUND_UP(sp->max_read_write_size, PAGE_SIZE) + 1; 2089 if (pages_per_rw > sc->ib.dev->attrs.max_sgl_rd) { 2090 ret = ib_mr_pool_init(sc->ib.qp, &sc->ib.qp->rdma_mrs, 2091 sc->rw_io.credits.max, IB_MR_TYPE_MEM_REG, 2092 sc->rw_io.credits.num_pages, 0); 2093 if (ret) { 2094 pr_err("failed to init mr pool count %zu pages %zu\n", 2095 sc->rw_io.credits.max, sc->rw_io.credits.num_pages); 2096 goto err; 2097 } 2098 } 2099 2100 return 0; 2101 err: 2102 if (sc->ib.qp) { 2103 sc->ib.qp = NULL; 2104 rdma_destroy_qp(sc->rdma.cm_id); 2105 } 2106 if (sc->ib.recv_cq) { 2107 ib_destroy_cq(sc->ib.recv_cq); 2108 sc->ib.recv_cq = NULL; 2109 } 2110 if (sc->ib.send_cq) { 2111 ib_destroy_cq(sc->ib.send_cq); 2112 sc->ib.send_cq = NULL; 2113 } 2114 if (sc->ib.pd) { 2115 ib_dealloc_pd(sc->ib.pd); 2116 sc->ib.pd = NULL; 2117 } 2118 return ret; 2119 } 2120 2121 static int smb_direct_prepare(struct ksmbd_transport *t) 2122 { 2123 struct smb_direct_transport *st = SMBD_TRANS(t); 2124 struct smbdirect_socket *sc = &st->socket; 2125 struct smbdirect_socket_parameters *sp = &sc->parameters; 2126 struct smbdirect_recv_io *recvmsg; 2127 struct smbdirect_negotiate_req *req; 2128 unsigned long flags; 2129 int ret; 2130 2131 /* 2132 * We are waiting to pass the following states: 2133 * 2134 * SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED 2135 * SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING 2136 * SMBDIRECT_SOCKET_NEGOTIATE_NEEDED 2137 * 2138 * To finally get to SMBDIRECT_SOCKET_NEGOTIATE_RUNNING 2139 * in order to continue below. 2140 * 2141 * Everything else is unexpected and an error. 2142 */ 2143 ksmbd_debug(RDMA, "Waiting for SMB_DIRECT negotiate request\n"); 2144 ret = wait_event_interruptible_timeout(sc->status_wait, 2145 sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED && 2146 sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING && 2147 sc->status != SMBDIRECT_SOCKET_NEGOTIATE_NEEDED, 2148 msecs_to_jiffies(sp->negotiate_timeout_msec)); 2149 if (ret <= 0 || sc->status != SMBDIRECT_SOCKET_NEGOTIATE_RUNNING) 2150 return ret < 0 ? ret : -ETIMEDOUT; 2151 2152 recvmsg = get_first_reassembly(sc); 2153 if (!recvmsg) 2154 return -ECONNABORTED; 2155 2156 ret = smb_direct_check_recvmsg(recvmsg); 2157 if (ret == -ECONNABORTED) 2158 goto out; 2159 2160 req = (struct smbdirect_negotiate_req *)recvmsg->packet; 2161 sp->max_recv_size = min_t(int, sp->max_recv_size, 2162 le32_to_cpu(req->preferred_send_size)); 2163 sp->max_send_size = min_t(int, sp->max_send_size, 2164 le32_to_cpu(req->max_receive_size)); 2165 sp->max_fragmented_send_size = 2166 le32_to_cpu(req->max_fragmented_size); 2167 sp->max_fragmented_recv_size = 2168 (sp->recv_credit_max * sp->max_recv_size) / 2; 2169 sc->recv_io.credits.target = le16_to_cpu(req->credits_requested); 2170 sc->recv_io.credits.target = min_t(u16, sc->recv_io.credits.target, sp->recv_credit_max); 2171 sc->recv_io.credits.target = max_t(u16, sc->recv_io.credits.target, 1); 2172 2173 ret = smb_direct_send_negotiate_response(sc, ret); 2174 out: 2175 spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); 2176 sc->recv_io.reassembly.queue_length--; 2177 list_del(&recvmsg->list); 2178 spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); 2179 put_recvmsg(sc, recvmsg); 2180 2181 return ret; 2182 } 2183 2184 static int smb_direct_connect(struct smbdirect_socket *sc) 2185 { 2186 struct ib_qp_cap qp_cap; 2187 int ret; 2188 2189 ret = smb_direct_init_params(sc, &qp_cap); 2190 if (ret) { 2191 pr_err("Can't configure RDMA parameters\n"); 2192 return ret; 2193 } 2194 2195 ret = smb_direct_create_pools(sc); 2196 if (ret) { 2197 pr_err("Can't init RDMA pool: %d\n", ret); 2198 return ret; 2199 } 2200 2201 ret = smb_direct_create_qpair(sc, &qp_cap); 2202 if (ret) { 2203 pr_err("Can't accept RDMA client: %d\n", ret); 2204 return ret; 2205 } 2206 2207 ret = smb_direct_prepare_negotiation(sc); 2208 if (ret) { 2209 pr_err("Can't negotiate: %d\n", ret); 2210 return ret; 2211 } 2212 return 0; 2213 } 2214 2215 static bool rdma_frwr_is_supported(struct ib_device_attr *attrs) 2216 { 2217 if (!(attrs->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)) 2218 return false; 2219 if (attrs->max_fast_reg_page_list_len == 0) 2220 return false; 2221 return true; 2222 } 2223 2224 static int smb_direct_handle_connect_request(struct rdma_cm_id *new_cm_id, 2225 struct rdma_cm_event *event) 2226 { 2227 struct smb_direct_transport *t; 2228 struct smbdirect_socket *sc; 2229 struct smbdirect_socket_parameters *sp; 2230 struct task_struct *handler; 2231 u8 peer_initiator_depth; 2232 u8 peer_responder_resources; 2233 int ret; 2234 2235 if (!rdma_frwr_is_supported(&new_cm_id->device->attrs)) { 2236 ksmbd_debug(RDMA, 2237 "Fast Registration Work Requests is not supported. device capabilities=%llx\n", 2238 new_cm_id->device->attrs.device_cap_flags); 2239 return -EPROTONOSUPPORT; 2240 } 2241 2242 t = alloc_transport(new_cm_id); 2243 if (!t) 2244 return -ENOMEM; 2245 sc = &t->socket; 2246 sp = &sc->parameters; 2247 2248 peer_initiator_depth = event->param.conn.initiator_depth; 2249 peer_responder_resources = event->param.conn.responder_resources; 2250 if (rdma_protocol_iwarp(new_cm_id->device, new_cm_id->port_num) && 2251 event->param.conn.private_data_len == 8) { 2252 /* 2253 * Legacy clients with only iWarp MPA v1 support 2254 * need a private blob in order to negotiate 2255 * the IRD/ORD values. 2256 */ 2257 const __be32 *ird_ord_hdr = event->param.conn.private_data; 2258 u32 ird32 = be32_to_cpu(ird_ord_hdr[0]); 2259 u32 ord32 = be32_to_cpu(ird_ord_hdr[1]); 2260 2261 /* 2262 * cifs.ko sends the legacy IRD/ORD negotiation 2263 * event if iWarp MPA v2 was used. 2264 * 2265 * Here we check that the values match and only 2266 * mark the client as legacy if they don't match. 2267 */ 2268 if ((u32)event->param.conn.initiator_depth != ird32 || 2269 (u32)event->param.conn.responder_resources != ord32) { 2270 /* 2271 * There are broken clients (old cifs.ko) 2272 * using little endian and also 2273 * struct rdma_conn_param only uses u8 2274 * for initiator_depth and responder_resources, 2275 * so we truncate the value to U8_MAX. 2276 * 2277 * smb_direct_accept_client() will then 2278 * do the real negotiation in order to 2279 * select the minimum between client and 2280 * server. 2281 */ 2282 ird32 = min_t(u32, ird32, U8_MAX); 2283 ord32 = min_t(u32, ord32, U8_MAX); 2284 2285 sc->rdma.legacy_iwarp = true; 2286 peer_initiator_depth = (u8)ird32; 2287 peer_responder_resources = (u8)ord32; 2288 } 2289 } 2290 2291 /* 2292 * First set what the we as server are able to support 2293 */ 2294 sp->initiator_depth = min_t(u8, sp->initiator_depth, 2295 new_cm_id->device->attrs.max_qp_rd_atom); 2296 2297 /* 2298 * negotiate the value by using the minimum 2299 * between client and server if the client provided 2300 * non 0 values. 2301 */ 2302 if (peer_initiator_depth != 0) 2303 sp->initiator_depth = min_t(u8, sp->initiator_depth, 2304 peer_initiator_depth); 2305 if (peer_responder_resources != 0) 2306 sp->responder_resources = min_t(u8, sp->responder_resources, 2307 peer_responder_resources); 2308 2309 ret = smb_direct_connect(sc); 2310 if (ret) 2311 goto out_err; 2312 2313 handler = kthread_run(ksmbd_conn_handler_loop, 2314 KSMBD_TRANS(t)->conn, "ksmbd:r%u", 2315 smb_direct_port); 2316 if (IS_ERR(handler)) { 2317 ret = PTR_ERR(handler); 2318 pr_err("Can't start thread\n"); 2319 goto out_err; 2320 } 2321 2322 return 0; 2323 out_err: 2324 free_transport(t); 2325 return ret; 2326 } 2327 2328 static int smb_direct_listen_handler(struct rdma_cm_id *cm_id, 2329 struct rdma_cm_event *event) 2330 { 2331 switch (event->event) { 2332 case RDMA_CM_EVENT_CONNECT_REQUEST: { 2333 int ret = smb_direct_handle_connect_request(cm_id, event); 2334 2335 if (ret) { 2336 pr_err("Can't create transport: %d\n", ret); 2337 return ret; 2338 } 2339 2340 ksmbd_debug(RDMA, "Received connection request. cm_id=%p\n", 2341 cm_id); 2342 break; 2343 } 2344 default: 2345 pr_err("Unexpected listen event. cm_id=%p, event=%s (%d)\n", 2346 cm_id, rdma_event_msg(event->event), event->event); 2347 break; 2348 } 2349 return 0; 2350 } 2351 2352 static int smb_direct_listen(int port) 2353 { 2354 int ret; 2355 struct rdma_cm_id *cm_id; 2356 struct sockaddr_in sin = { 2357 .sin_family = AF_INET, 2358 .sin_addr.s_addr = htonl(INADDR_ANY), 2359 .sin_port = htons(port), 2360 }; 2361 2362 cm_id = rdma_create_id(&init_net, smb_direct_listen_handler, 2363 &smb_direct_listener, RDMA_PS_TCP, IB_QPT_RC); 2364 if (IS_ERR(cm_id)) { 2365 pr_err("Can't create cm id: %ld\n", PTR_ERR(cm_id)); 2366 return PTR_ERR(cm_id); 2367 } 2368 2369 ret = rdma_bind_addr(cm_id, (struct sockaddr *)&sin); 2370 if (ret) { 2371 pr_err("Can't bind: %d\n", ret); 2372 goto err; 2373 } 2374 2375 smb_direct_listener.cm_id = cm_id; 2376 2377 ret = rdma_listen(cm_id, 10); 2378 if (ret) { 2379 pr_err("Can't listen: %d\n", ret); 2380 goto err; 2381 } 2382 return 0; 2383 err: 2384 smb_direct_listener.cm_id = NULL; 2385 rdma_destroy_id(cm_id); 2386 return ret; 2387 } 2388 2389 static int smb_direct_ib_client_add(struct ib_device *ib_dev) 2390 { 2391 struct smb_direct_device *smb_dev; 2392 2393 /* Set 5445 port if device type is iWARP(No IB) */ 2394 if (ib_dev->node_type != RDMA_NODE_IB_CA) 2395 smb_direct_port = SMB_DIRECT_PORT_IWARP; 2396 2397 if (!rdma_frwr_is_supported(&ib_dev->attrs)) 2398 return 0; 2399 2400 smb_dev = kzalloc(sizeof(*smb_dev), KSMBD_DEFAULT_GFP); 2401 if (!smb_dev) 2402 return -ENOMEM; 2403 smb_dev->ib_dev = ib_dev; 2404 2405 write_lock(&smb_direct_device_lock); 2406 list_add(&smb_dev->list, &smb_direct_device_list); 2407 write_unlock(&smb_direct_device_lock); 2408 2409 ksmbd_debug(RDMA, "ib device added: name %s\n", ib_dev->name); 2410 return 0; 2411 } 2412 2413 static void smb_direct_ib_client_remove(struct ib_device *ib_dev, 2414 void *client_data) 2415 { 2416 struct smb_direct_device *smb_dev, *tmp; 2417 2418 write_lock(&smb_direct_device_lock); 2419 list_for_each_entry_safe(smb_dev, tmp, &smb_direct_device_list, list) { 2420 if (smb_dev->ib_dev == ib_dev) { 2421 list_del(&smb_dev->list); 2422 kfree(smb_dev); 2423 break; 2424 } 2425 } 2426 write_unlock(&smb_direct_device_lock); 2427 } 2428 2429 static struct ib_client smb_direct_ib_client = { 2430 .name = "ksmbd_smb_direct_ib", 2431 .add = smb_direct_ib_client_add, 2432 .remove = smb_direct_ib_client_remove, 2433 }; 2434 2435 int ksmbd_rdma_init(void) 2436 { 2437 int ret; 2438 2439 smb_direct_listener.cm_id = NULL; 2440 2441 ret = ib_register_client(&smb_direct_ib_client); 2442 if (ret) { 2443 pr_err("failed to ib_register_client\n"); 2444 return ret; 2445 } 2446 2447 /* When a client is running out of send credits, the credits are 2448 * granted by the server's sending a packet using this queue. 2449 * This avoids the situation that a clients cannot send packets 2450 * for lack of credits 2451 */ 2452 smb_direct_wq = alloc_workqueue("ksmbd-smb_direct-wq", 2453 WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_PERCPU, 2454 0); 2455 if (!smb_direct_wq) 2456 return -ENOMEM; 2457 2458 ret = smb_direct_listen(smb_direct_port); 2459 if (ret) { 2460 destroy_workqueue(smb_direct_wq); 2461 smb_direct_wq = NULL; 2462 pr_err("Can't listen: %d\n", ret); 2463 return ret; 2464 } 2465 2466 ksmbd_debug(RDMA, "init RDMA listener. cm_id=%p\n", 2467 smb_direct_listener.cm_id); 2468 return 0; 2469 } 2470 2471 void ksmbd_rdma_stop_listening(void) 2472 { 2473 if (!smb_direct_listener.cm_id) 2474 return; 2475 2476 ib_unregister_client(&smb_direct_ib_client); 2477 rdma_destroy_id(smb_direct_listener.cm_id); 2478 2479 smb_direct_listener.cm_id = NULL; 2480 } 2481 2482 void ksmbd_rdma_destroy(void) 2483 { 2484 if (smb_direct_wq) { 2485 destroy_workqueue(smb_direct_wq); 2486 smb_direct_wq = NULL; 2487 } 2488 } 2489 2490 bool ksmbd_rdma_capable_netdev(struct net_device *netdev) 2491 { 2492 struct smb_direct_device *smb_dev; 2493 int i; 2494 bool rdma_capable = false; 2495 2496 read_lock(&smb_direct_device_lock); 2497 list_for_each_entry(smb_dev, &smb_direct_device_list, list) { 2498 for (i = 0; i < smb_dev->ib_dev->phys_port_cnt; i++) { 2499 struct net_device *ndev; 2500 2501 ndev = ib_device_get_netdev(smb_dev->ib_dev, i + 1); 2502 if (!ndev) 2503 continue; 2504 2505 if (ndev == netdev) { 2506 dev_put(ndev); 2507 rdma_capable = true; 2508 goto out; 2509 } 2510 dev_put(ndev); 2511 } 2512 } 2513 out: 2514 read_unlock(&smb_direct_device_lock); 2515 2516 if (rdma_capable == false) { 2517 struct ib_device *ibdev; 2518 2519 ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_UNKNOWN); 2520 if (ibdev) { 2521 rdma_capable = rdma_frwr_is_supported(&ibdev->attrs); 2522 ib_device_put(ibdev); 2523 } 2524 } 2525 2526 ksmbd_debug(RDMA, "netdev(%s) rdma capable : %s\n", 2527 netdev->name, str_true_false(rdma_capable)); 2528 2529 return rdma_capable; 2530 } 2531 2532 static const struct ksmbd_transport_ops ksmbd_smb_direct_transport_ops = { 2533 .prepare = smb_direct_prepare, 2534 .disconnect = smb_direct_disconnect, 2535 .shutdown = smb_direct_shutdown, 2536 .writev = smb_direct_writev, 2537 .read = smb_direct_read, 2538 .rdma_read = smb_direct_rdma_read, 2539 .rdma_write = smb_direct_rdma_write, 2540 .free_transport = smb_direct_free_transport, 2541 }; 2542