1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Copyright (C) 2017, Microsoft Corporation.
4 * Copyright (C) 2018, LG Electronics.
5 *
6 * Author(s): Long Li <longli@microsoft.com>,
7 * Hyunchul Lee <hyc.lee@gmail.com>
8 */
9
10 #define SUBMOD_NAME "smb_direct"
11
12 #include <linux/kthread.h>
13 #include <linux/list.h>
14 #include <linux/mempool.h>
15 #include <linux/highmem.h>
16 #include <linux/scatterlist.h>
17 #include <rdma/ib_verbs.h>
18 #include <rdma/rdma_cm.h>
19 #include <rdma/rw.h>
20
21 #include "glob.h"
22 #include "connection.h"
23 #include "smb_common.h"
24 #include "../common/smb2status.h"
25 #include "transport_rdma.h"
26
27 #define SMB_DIRECT_PORT_IWARP 5445
28 #define SMB_DIRECT_PORT_INFINIBAND 445
29
30 #define SMB_DIRECT_VERSION_LE cpu_to_le16(0x0100)
31
32 /* SMB_DIRECT negotiation timeout in seconds */
33 #define SMB_DIRECT_NEGOTIATE_TIMEOUT 120
34
35 #define SMB_DIRECT_MAX_SEND_SGES 6
36 #define SMB_DIRECT_MAX_RECV_SGES 1
37
38 /*
39 * Default maximum number of RDMA read/write outstanding on this connection
40 * This value is possibly decreased during QP creation on hardware limit
41 */
42 #define SMB_DIRECT_CM_INITIATOR_DEPTH 8
43
44 /* Maximum number of retries on data transfer operations */
45 #define SMB_DIRECT_CM_RETRY 6
46 /* No need to retry on Receiver Not Ready since SMB_DIRECT manages credits */
47 #define SMB_DIRECT_CM_RNR_RETRY 0
48
49 /*
50 * User configurable initial values per SMB_DIRECT transport connection
51 * as defined in [MS-SMBD] 3.1.1.1
52 * Those may change after a SMB_DIRECT negotiation
53 */
54
55 /* Set 445 port to SMB Direct port by default */
56 static int smb_direct_port = SMB_DIRECT_PORT_INFINIBAND;
57
58 /* The local peer's maximum number of credits to grant to the peer */
59 static int smb_direct_receive_credit_max = 255;
60
61 /* The remote peer's credit request of local peer */
62 static int smb_direct_send_credit_target = 255;
63
64 /* The maximum single message size can be sent to remote peer */
65 static int smb_direct_max_send_size = 1364;
66
67 /* The maximum fragmented upper-layer payload receive size supported */
68 static int smb_direct_max_fragmented_recv_size = 1024 * 1024;
69
70 /* The maximum single-message size which can be received */
71 static int smb_direct_max_receive_size = 1364;
72
73 static int smb_direct_max_read_write_size = SMBD_DEFAULT_IOSIZE;
74
75 static LIST_HEAD(smb_direct_device_list);
76 static DEFINE_RWLOCK(smb_direct_device_lock);
77
78 struct smb_direct_device {
79 struct ib_device *ib_dev;
80 struct list_head list;
81 };
82
83 static struct smb_direct_listener {
84 struct rdma_cm_id *cm_id;
85 } smb_direct_listener;
86
87 static struct workqueue_struct *smb_direct_wq;
88
89 enum smb_direct_status {
90 SMB_DIRECT_CS_NEW = 0,
91 SMB_DIRECT_CS_CONNECTED,
92 SMB_DIRECT_CS_DISCONNECTING,
93 SMB_DIRECT_CS_DISCONNECTED,
94 };
95
96 struct smb_direct_transport {
97 struct ksmbd_transport transport;
98
99 enum smb_direct_status status;
100 bool full_packet_received;
101 wait_queue_head_t wait_status;
102
103 struct rdma_cm_id *cm_id;
104 struct ib_cq *send_cq;
105 struct ib_cq *recv_cq;
106 struct ib_pd *pd;
107 struct ib_qp *qp;
108
109 int max_send_size;
110 int max_recv_size;
111 int max_fragmented_send_size;
112 int max_fragmented_recv_size;
113 int max_rdma_rw_size;
114
115 spinlock_t reassembly_queue_lock;
116 struct list_head reassembly_queue;
117 int reassembly_data_length;
118 int reassembly_queue_length;
119 int first_entry_offset;
120 wait_queue_head_t wait_reassembly_queue;
121
122 spinlock_t receive_credit_lock;
123 int recv_credits;
124 int count_avail_recvmsg;
125 int recv_credit_max;
126 int recv_credit_target;
127
128 spinlock_t recvmsg_queue_lock;
129 struct list_head recvmsg_queue;
130
131 spinlock_t empty_recvmsg_queue_lock;
132 struct list_head empty_recvmsg_queue;
133
134 int send_credit_target;
135 atomic_t send_credits;
136 spinlock_t lock_new_recv_credits;
137 int new_recv_credits;
138 int max_rw_credits;
139 int pages_per_rw_credit;
140 atomic_t rw_credits;
141
142 wait_queue_head_t wait_send_credits;
143 wait_queue_head_t wait_rw_credits;
144
145 mempool_t *sendmsg_mempool;
146 struct kmem_cache *sendmsg_cache;
147 mempool_t *recvmsg_mempool;
148 struct kmem_cache *recvmsg_cache;
149
150 wait_queue_head_t wait_send_pending;
151 atomic_t send_pending;
152
153 struct delayed_work post_recv_credits_work;
154 struct work_struct send_immediate_work;
155 struct work_struct disconnect_work;
156
157 bool negotiation_requested;
158 };
159
160 #define KSMBD_TRANS(t) ((struct ksmbd_transport *)&((t)->transport))
161
162 enum {
163 SMB_DIRECT_MSG_NEGOTIATE_REQ = 0,
164 SMB_DIRECT_MSG_DATA_TRANSFER
165 };
166
167 static const struct ksmbd_transport_ops ksmbd_smb_direct_transport_ops;
168
169 struct smb_direct_send_ctx {
170 struct list_head msg_list;
171 int wr_cnt;
172 bool need_invalidate_rkey;
173 unsigned int remote_key;
174 };
175
176 struct smb_direct_sendmsg {
177 struct smb_direct_transport *transport;
178 struct ib_send_wr wr;
179 struct list_head list;
180 int num_sge;
181 struct ib_sge sge[SMB_DIRECT_MAX_SEND_SGES];
182 struct ib_cqe cqe;
183 u8 packet[];
184 };
185
186 struct smb_direct_recvmsg {
187 struct smb_direct_transport *transport;
188 struct list_head list;
189 int type;
190 struct ib_sge sge;
191 struct ib_cqe cqe;
192 bool first_segment;
193 u8 packet[];
194 };
195
196 struct smb_direct_rdma_rw_msg {
197 struct smb_direct_transport *t;
198 struct ib_cqe cqe;
199 int status;
200 struct completion *completion;
201 struct list_head list;
202 struct rdma_rw_ctx rw_ctx;
203 struct sg_table sgt;
204 struct scatterlist sg_list[];
205 };
206
init_smbd_max_io_size(unsigned int sz)207 void init_smbd_max_io_size(unsigned int sz)
208 {
209 sz = clamp_val(sz, SMBD_MIN_IOSIZE, SMBD_MAX_IOSIZE);
210 smb_direct_max_read_write_size = sz;
211 }
212
get_smbd_max_read_write_size(void)213 unsigned int get_smbd_max_read_write_size(void)
214 {
215 return smb_direct_max_read_write_size;
216 }
217
get_buf_page_count(void * buf,int size)218 static inline int get_buf_page_count(void *buf, int size)
219 {
220 return DIV_ROUND_UP((uintptr_t)buf + size, PAGE_SIZE) -
221 (uintptr_t)buf / PAGE_SIZE;
222 }
223
224 static void smb_direct_destroy_pools(struct smb_direct_transport *transport);
225 static void smb_direct_post_recv_credits(struct work_struct *work);
226 static int smb_direct_post_send_data(struct smb_direct_transport *t,
227 struct smb_direct_send_ctx *send_ctx,
228 struct kvec *iov, int niov,
229 int remaining_data_length);
230
231 static inline struct smb_direct_transport *
smb_trans_direct_transfort(struct ksmbd_transport * t)232 smb_trans_direct_transfort(struct ksmbd_transport *t)
233 {
234 return container_of(t, struct smb_direct_transport, transport);
235 }
236
237 static inline void
smb_direct_recvmsg_payload(struct smb_direct_recvmsg * recvmsg)238 *smb_direct_recvmsg_payload(struct smb_direct_recvmsg *recvmsg)
239 {
240 return (void *)recvmsg->packet;
241 }
242
is_receive_credit_post_required(int receive_credits,int avail_recvmsg_count)243 static inline bool is_receive_credit_post_required(int receive_credits,
244 int avail_recvmsg_count)
245 {
246 return receive_credits <= (smb_direct_receive_credit_max >> 3) &&
247 avail_recvmsg_count >= (receive_credits >> 2);
248 }
249
250 static struct
get_free_recvmsg(struct smb_direct_transport * t)251 smb_direct_recvmsg *get_free_recvmsg(struct smb_direct_transport *t)
252 {
253 struct smb_direct_recvmsg *recvmsg = NULL;
254
255 spin_lock(&t->recvmsg_queue_lock);
256 if (!list_empty(&t->recvmsg_queue)) {
257 recvmsg = list_first_entry(&t->recvmsg_queue,
258 struct smb_direct_recvmsg,
259 list);
260 list_del(&recvmsg->list);
261 }
262 spin_unlock(&t->recvmsg_queue_lock);
263 return recvmsg;
264 }
265
put_recvmsg(struct smb_direct_transport * t,struct smb_direct_recvmsg * recvmsg)266 static void put_recvmsg(struct smb_direct_transport *t,
267 struct smb_direct_recvmsg *recvmsg)
268 {
269 ib_dma_unmap_single(t->cm_id->device, recvmsg->sge.addr,
270 recvmsg->sge.length, DMA_FROM_DEVICE);
271
272 spin_lock(&t->recvmsg_queue_lock);
273 list_add(&recvmsg->list, &t->recvmsg_queue);
274 spin_unlock(&t->recvmsg_queue_lock);
275 }
276
277 static struct
get_empty_recvmsg(struct smb_direct_transport * t)278 smb_direct_recvmsg *get_empty_recvmsg(struct smb_direct_transport *t)
279 {
280 struct smb_direct_recvmsg *recvmsg = NULL;
281
282 spin_lock(&t->empty_recvmsg_queue_lock);
283 if (!list_empty(&t->empty_recvmsg_queue)) {
284 recvmsg = list_first_entry(&t->empty_recvmsg_queue,
285 struct smb_direct_recvmsg, list);
286 list_del(&recvmsg->list);
287 }
288 spin_unlock(&t->empty_recvmsg_queue_lock);
289 return recvmsg;
290 }
291
put_empty_recvmsg(struct smb_direct_transport * t,struct smb_direct_recvmsg * recvmsg)292 static void put_empty_recvmsg(struct smb_direct_transport *t,
293 struct smb_direct_recvmsg *recvmsg)
294 {
295 ib_dma_unmap_single(t->cm_id->device, recvmsg->sge.addr,
296 recvmsg->sge.length, DMA_FROM_DEVICE);
297
298 spin_lock(&t->empty_recvmsg_queue_lock);
299 list_add_tail(&recvmsg->list, &t->empty_recvmsg_queue);
300 spin_unlock(&t->empty_recvmsg_queue_lock);
301 }
302
enqueue_reassembly(struct smb_direct_transport * t,struct smb_direct_recvmsg * recvmsg,int data_length)303 static void enqueue_reassembly(struct smb_direct_transport *t,
304 struct smb_direct_recvmsg *recvmsg,
305 int data_length)
306 {
307 spin_lock(&t->reassembly_queue_lock);
308 list_add_tail(&recvmsg->list, &t->reassembly_queue);
309 t->reassembly_queue_length++;
310 /*
311 * Make sure reassembly_data_length is updated after list and
312 * reassembly_queue_length are updated. On the dequeue side
313 * reassembly_data_length is checked without a lock to determine
314 * if reassembly_queue_length and list is up to date
315 */
316 virt_wmb();
317 t->reassembly_data_length += data_length;
318 spin_unlock(&t->reassembly_queue_lock);
319 }
320
get_first_reassembly(struct smb_direct_transport * t)321 static struct smb_direct_recvmsg *get_first_reassembly(struct smb_direct_transport *t)
322 {
323 if (!list_empty(&t->reassembly_queue))
324 return list_first_entry(&t->reassembly_queue,
325 struct smb_direct_recvmsg, list);
326 else
327 return NULL;
328 }
329
smb_direct_disconnect_rdma_work(struct work_struct * work)330 static void smb_direct_disconnect_rdma_work(struct work_struct *work)
331 {
332 struct smb_direct_transport *t =
333 container_of(work, struct smb_direct_transport,
334 disconnect_work);
335
336 if (t->status == SMB_DIRECT_CS_CONNECTED) {
337 t->status = SMB_DIRECT_CS_DISCONNECTING;
338 rdma_disconnect(t->cm_id);
339 }
340 }
341
342 static void
smb_direct_disconnect_rdma_connection(struct smb_direct_transport * t)343 smb_direct_disconnect_rdma_connection(struct smb_direct_transport *t)
344 {
345 if (t->status == SMB_DIRECT_CS_CONNECTED)
346 queue_work(smb_direct_wq, &t->disconnect_work);
347 }
348
smb_direct_send_immediate_work(struct work_struct * work)349 static void smb_direct_send_immediate_work(struct work_struct *work)
350 {
351 struct smb_direct_transport *t = container_of(work,
352 struct smb_direct_transport, send_immediate_work);
353
354 if (t->status != SMB_DIRECT_CS_CONNECTED)
355 return;
356
357 smb_direct_post_send_data(t, NULL, NULL, 0, 0);
358 }
359
alloc_transport(struct rdma_cm_id * cm_id)360 static struct smb_direct_transport *alloc_transport(struct rdma_cm_id *cm_id)
361 {
362 struct smb_direct_transport *t;
363 struct ksmbd_conn *conn;
364
365 t = kzalloc(sizeof(*t), GFP_KERNEL);
366 if (!t)
367 return NULL;
368
369 t->cm_id = cm_id;
370 cm_id->context = t;
371
372 t->status = SMB_DIRECT_CS_NEW;
373 init_waitqueue_head(&t->wait_status);
374
375 spin_lock_init(&t->reassembly_queue_lock);
376 INIT_LIST_HEAD(&t->reassembly_queue);
377 t->reassembly_data_length = 0;
378 t->reassembly_queue_length = 0;
379 init_waitqueue_head(&t->wait_reassembly_queue);
380 init_waitqueue_head(&t->wait_send_credits);
381 init_waitqueue_head(&t->wait_rw_credits);
382
383 spin_lock_init(&t->receive_credit_lock);
384 spin_lock_init(&t->recvmsg_queue_lock);
385 INIT_LIST_HEAD(&t->recvmsg_queue);
386
387 spin_lock_init(&t->empty_recvmsg_queue_lock);
388 INIT_LIST_HEAD(&t->empty_recvmsg_queue);
389
390 init_waitqueue_head(&t->wait_send_pending);
391 atomic_set(&t->send_pending, 0);
392
393 spin_lock_init(&t->lock_new_recv_credits);
394
395 INIT_DELAYED_WORK(&t->post_recv_credits_work,
396 smb_direct_post_recv_credits);
397 INIT_WORK(&t->send_immediate_work, smb_direct_send_immediate_work);
398 INIT_WORK(&t->disconnect_work, smb_direct_disconnect_rdma_work);
399
400 conn = ksmbd_conn_alloc();
401 if (!conn)
402 goto err;
403 conn->transport = KSMBD_TRANS(t);
404 KSMBD_TRANS(t)->conn = conn;
405 KSMBD_TRANS(t)->ops = &ksmbd_smb_direct_transport_ops;
406 return t;
407 err:
408 kfree(t);
409 return NULL;
410 }
411
free_transport(struct smb_direct_transport * t)412 static void free_transport(struct smb_direct_transport *t)
413 {
414 struct smb_direct_recvmsg *recvmsg;
415
416 wake_up_interruptible(&t->wait_send_credits);
417
418 ksmbd_debug(RDMA, "wait for all send posted to IB to finish\n");
419 wait_event(t->wait_send_pending,
420 atomic_read(&t->send_pending) == 0);
421
422 cancel_work_sync(&t->disconnect_work);
423 cancel_delayed_work_sync(&t->post_recv_credits_work);
424 cancel_work_sync(&t->send_immediate_work);
425
426 if (t->qp) {
427 ib_drain_qp(t->qp);
428 ib_mr_pool_destroy(t->qp, &t->qp->rdma_mrs);
429 ib_destroy_qp(t->qp);
430 }
431
432 ksmbd_debug(RDMA, "drain the reassembly queue\n");
433 do {
434 spin_lock(&t->reassembly_queue_lock);
435 recvmsg = get_first_reassembly(t);
436 if (recvmsg) {
437 list_del(&recvmsg->list);
438 spin_unlock(&t->reassembly_queue_lock);
439 put_recvmsg(t, recvmsg);
440 } else {
441 spin_unlock(&t->reassembly_queue_lock);
442 }
443 } while (recvmsg);
444 t->reassembly_data_length = 0;
445
446 if (t->send_cq)
447 ib_free_cq(t->send_cq);
448 if (t->recv_cq)
449 ib_free_cq(t->recv_cq);
450 if (t->pd)
451 ib_dealloc_pd(t->pd);
452 if (t->cm_id)
453 rdma_destroy_id(t->cm_id);
454
455 smb_direct_destroy_pools(t);
456 ksmbd_conn_free(KSMBD_TRANS(t)->conn);
457 kfree(t);
458 }
459
460 static struct smb_direct_sendmsg
smb_direct_alloc_sendmsg(struct smb_direct_transport * t)461 *smb_direct_alloc_sendmsg(struct smb_direct_transport *t)
462 {
463 struct smb_direct_sendmsg *msg;
464
465 msg = mempool_alloc(t->sendmsg_mempool, GFP_KERNEL);
466 if (!msg)
467 return ERR_PTR(-ENOMEM);
468 msg->transport = t;
469 INIT_LIST_HEAD(&msg->list);
470 msg->num_sge = 0;
471 return msg;
472 }
473
smb_direct_free_sendmsg(struct smb_direct_transport * t,struct smb_direct_sendmsg * msg)474 static void smb_direct_free_sendmsg(struct smb_direct_transport *t,
475 struct smb_direct_sendmsg *msg)
476 {
477 int i;
478
479 if (msg->num_sge > 0) {
480 ib_dma_unmap_single(t->cm_id->device,
481 msg->sge[0].addr, msg->sge[0].length,
482 DMA_TO_DEVICE);
483 for (i = 1; i < msg->num_sge; i++)
484 ib_dma_unmap_page(t->cm_id->device,
485 msg->sge[i].addr, msg->sge[i].length,
486 DMA_TO_DEVICE);
487 }
488 mempool_free(msg, t->sendmsg_mempool);
489 }
490
smb_direct_check_recvmsg(struct smb_direct_recvmsg * recvmsg)491 static int smb_direct_check_recvmsg(struct smb_direct_recvmsg *recvmsg)
492 {
493 switch (recvmsg->type) {
494 case SMB_DIRECT_MSG_DATA_TRANSFER: {
495 struct smb_direct_data_transfer *req =
496 (struct smb_direct_data_transfer *)recvmsg->packet;
497 struct smb2_hdr *hdr = (struct smb2_hdr *)(recvmsg->packet
498 + le32_to_cpu(req->data_offset));
499 ksmbd_debug(RDMA,
500 "CreditGranted: %u, CreditRequested: %u, DataLength: %u, RemainingDataLength: %u, SMB: %x, Command: %u\n",
501 le16_to_cpu(req->credits_granted),
502 le16_to_cpu(req->credits_requested),
503 req->data_length, req->remaining_data_length,
504 hdr->ProtocolId, hdr->Command);
505 break;
506 }
507 case SMB_DIRECT_MSG_NEGOTIATE_REQ: {
508 struct smb_direct_negotiate_req *req =
509 (struct smb_direct_negotiate_req *)recvmsg->packet;
510 ksmbd_debug(RDMA,
511 "MinVersion: %u, MaxVersion: %u, CreditRequested: %u, MaxSendSize: %u, MaxRecvSize: %u, MaxFragmentedSize: %u\n",
512 le16_to_cpu(req->min_version),
513 le16_to_cpu(req->max_version),
514 le16_to_cpu(req->credits_requested),
515 le32_to_cpu(req->preferred_send_size),
516 le32_to_cpu(req->max_receive_size),
517 le32_to_cpu(req->max_fragmented_size));
518 if (le16_to_cpu(req->min_version) > 0x0100 ||
519 le16_to_cpu(req->max_version) < 0x0100)
520 return -EOPNOTSUPP;
521 if (le16_to_cpu(req->credits_requested) <= 0 ||
522 le32_to_cpu(req->max_receive_size) <= 128 ||
523 le32_to_cpu(req->max_fragmented_size) <=
524 128 * 1024)
525 return -ECONNABORTED;
526
527 break;
528 }
529 default:
530 return -EINVAL;
531 }
532 return 0;
533 }
534
recv_done(struct ib_cq * cq,struct ib_wc * wc)535 static void recv_done(struct ib_cq *cq, struct ib_wc *wc)
536 {
537 struct smb_direct_recvmsg *recvmsg;
538 struct smb_direct_transport *t;
539
540 recvmsg = container_of(wc->wr_cqe, struct smb_direct_recvmsg, cqe);
541 t = recvmsg->transport;
542
543 if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_RECV) {
544 if (wc->status != IB_WC_WR_FLUSH_ERR) {
545 pr_err("Recv error. status='%s (%d)' opcode=%d\n",
546 ib_wc_status_msg(wc->status), wc->status,
547 wc->opcode);
548 smb_direct_disconnect_rdma_connection(t);
549 }
550 put_empty_recvmsg(t, recvmsg);
551 return;
552 }
553
554 ksmbd_debug(RDMA, "Recv completed. status='%s (%d)', opcode=%d\n",
555 ib_wc_status_msg(wc->status), wc->status,
556 wc->opcode);
557
558 ib_dma_sync_single_for_cpu(wc->qp->device, recvmsg->sge.addr,
559 recvmsg->sge.length, DMA_FROM_DEVICE);
560
561 switch (recvmsg->type) {
562 case SMB_DIRECT_MSG_NEGOTIATE_REQ:
563 if (wc->byte_len < sizeof(struct smb_direct_negotiate_req)) {
564 put_empty_recvmsg(t, recvmsg);
565 return;
566 }
567 t->negotiation_requested = true;
568 t->full_packet_received = true;
569 t->status = SMB_DIRECT_CS_CONNECTED;
570 enqueue_reassembly(t, recvmsg, 0);
571 wake_up_interruptible(&t->wait_status);
572 break;
573 case SMB_DIRECT_MSG_DATA_TRANSFER: {
574 struct smb_direct_data_transfer *data_transfer =
575 (struct smb_direct_data_transfer *)recvmsg->packet;
576 unsigned int data_length;
577 int avail_recvmsg_count, receive_credits;
578
579 if (wc->byte_len <
580 offsetof(struct smb_direct_data_transfer, padding)) {
581 put_empty_recvmsg(t, recvmsg);
582 return;
583 }
584
585 data_length = le32_to_cpu(data_transfer->data_length);
586 if (data_length) {
587 if (wc->byte_len < sizeof(struct smb_direct_data_transfer) +
588 (u64)data_length) {
589 put_empty_recvmsg(t, recvmsg);
590 return;
591 }
592
593 if (t->full_packet_received)
594 recvmsg->first_segment = true;
595
596 if (le32_to_cpu(data_transfer->remaining_data_length))
597 t->full_packet_received = false;
598 else
599 t->full_packet_received = true;
600
601 enqueue_reassembly(t, recvmsg, (int)data_length);
602 wake_up_interruptible(&t->wait_reassembly_queue);
603
604 spin_lock(&t->receive_credit_lock);
605 receive_credits = --(t->recv_credits);
606 avail_recvmsg_count = t->count_avail_recvmsg;
607 spin_unlock(&t->receive_credit_lock);
608 } else {
609 put_empty_recvmsg(t, recvmsg);
610
611 spin_lock(&t->receive_credit_lock);
612 receive_credits = --(t->recv_credits);
613 avail_recvmsg_count = ++(t->count_avail_recvmsg);
614 spin_unlock(&t->receive_credit_lock);
615 }
616
617 t->recv_credit_target =
618 le16_to_cpu(data_transfer->credits_requested);
619 atomic_add(le16_to_cpu(data_transfer->credits_granted),
620 &t->send_credits);
621
622 if (le16_to_cpu(data_transfer->flags) &
623 SMB_DIRECT_RESPONSE_REQUESTED)
624 queue_work(smb_direct_wq, &t->send_immediate_work);
625
626 if (atomic_read(&t->send_credits) > 0)
627 wake_up_interruptible(&t->wait_send_credits);
628
629 if (is_receive_credit_post_required(receive_credits, avail_recvmsg_count))
630 mod_delayed_work(smb_direct_wq,
631 &t->post_recv_credits_work, 0);
632 break;
633 }
634 default:
635 break;
636 }
637 }
638
smb_direct_post_recv(struct smb_direct_transport * t,struct smb_direct_recvmsg * recvmsg)639 static int smb_direct_post_recv(struct smb_direct_transport *t,
640 struct smb_direct_recvmsg *recvmsg)
641 {
642 struct ib_recv_wr wr;
643 int ret;
644
645 recvmsg->sge.addr = ib_dma_map_single(t->cm_id->device,
646 recvmsg->packet, t->max_recv_size,
647 DMA_FROM_DEVICE);
648 ret = ib_dma_mapping_error(t->cm_id->device, recvmsg->sge.addr);
649 if (ret)
650 return ret;
651 recvmsg->sge.length = t->max_recv_size;
652 recvmsg->sge.lkey = t->pd->local_dma_lkey;
653 recvmsg->cqe.done = recv_done;
654
655 wr.wr_cqe = &recvmsg->cqe;
656 wr.next = NULL;
657 wr.sg_list = &recvmsg->sge;
658 wr.num_sge = 1;
659
660 ret = ib_post_recv(t->qp, &wr, NULL);
661 if (ret) {
662 pr_err("Can't post recv: %d\n", ret);
663 ib_dma_unmap_single(t->cm_id->device,
664 recvmsg->sge.addr, recvmsg->sge.length,
665 DMA_FROM_DEVICE);
666 smb_direct_disconnect_rdma_connection(t);
667 return ret;
668 }
669 return ret;
670 }
671
smb_direct_read(struct ksmbd_transport * t,char * buf,unsigned int size,int unused)672 static int smb_direct_read(struct ksmbd_transport *t, char *buf,
673 unsigned int size, int unused)
674 {
675 struct smb_direct_recvmsg *recvmsg;
676 struct smb_direct_data_transfer *data_transfer;
677 int to_copy, to_read, data_read, offset;
678 u32 data_length, remaining_data_length, data_offset;
679 int rc;
680 struct smb_direct_transport *st = smb_trans_direct_transfort(t);
681
682 again:
683 if (st->status != SMB_DIRECT_CS_CONNECTED) {
684 pr_err("disconnected\n");
685 return -ENOTCONN;
686 }
687
688 /*
689 * No need to hold the reassembly queue lock all the time as we are
690 * the only one reading from the front of the queue. The transport
691 * may add more entries to the back of the queue at the same time
692 */
693 if (st->reassembly_data_length >= size) {
694 int queue_length;
695 int queue_removed = 0;
696
697 /*
698 * Need to make sure reassembly_data_length is read before
699 * reading reassembly_queue_length and calling
700 * get_first_reassembly. This call is lock free
701 * as we never read at the end of the queue which are being
702 * updated in SOFTIRQ as more data is received
703 */
704 virt_rmb();
705 queue_length = st->reassembly_queue_length;
706 data_read = 0;
707 to_read = size;
708 offset = st->first_entry_offset;
709 while (data_read < size) {
710 recvmsg = get_first_reassembly(st);
711 data_transfer = smb_direct_recvmsg_payload(recvmsg);
712 data_length = le32_to_cpu(data_transfer->data_length);
713 remaining_data_length =
714 le32_to_cpu(data_transfer->remaining_data_length);
715 data_offset = le32_to_cpu(data_transfer->data_offset);
716
717 /*
718 * The upper layer expects RFC1002 length at the
719 * beginning of the payload. Return it to indicate
720 * the total length of the packet. This minimize the
721 * change to upper layer packet processing logic. This
722 * will be eventually remove when an intermediate
723 * transport layer is added
724 */
725 if (recvmsg->first_segment && size == 4) {
726 unsigned int rfc1002_len =
727 data_length + remaining_data_length;
728 *((__be32 *)buf) = cpu_to_be32(rfc1002_len);
729 data_read = 4;
730 recvmsg->first_segment = false;
731 ksmbd_debug(RDMA,
732 "returning rfc1002 length %d\n",
733 rfc1002_len);
734 goto read_rfc1002_done;
735 }
736
737 to_copy = min_t(int, data_length - offset, to_read);
738 memcpy(buf + data_read, (char *)data_transfer + data_offset + offset,
739 to_copy);
740
741 /* move on to the next buffer? */
742 if (to_copy == data_length - offset) {
743 queue_length--;
744 /*
745 * No need to lock if we are not at the
746 * end of the queue
747 */
748 if (queue_length) {
749 list_del(&recvmsg->list);
750 } else {
751 spin_lock_irq(&st->reassembly_queue_lock);
752 list_del(&recvmsg->list);
753 spin_unlock_irq(&st->reassembly_queue_lock);
754 }
755 queue_removed++;
756 put_recvmsg(st, recvmsg);
757 offset = 0;
758 } else {
759 offset += to_copy;
760 }
761
762 to_read -= to_copy;
763 data_read += to_copy;
764 }
765
766 spin_lock_irq(&st->reassembly_queue_lock);
767 st->reassembly_data_length -= data_read;
768 st->reassembly_queue_length -= queue_removed;
769 spin_unlock_irq(&st->reassembly_queue_lock);
770
771 spin_lock(&st->receive_credit_lock);
772 st->count_avail_recvmsg += queue_removed;
773 if (is_receive_credit_post_required(st->recv_credits, st->count_avail_recvmsg)) {
774 spin_unlock(&st->receive_credit_lock);
775 mod_delayed_work(smb_direct_wq,
776 &st->post_recv_credits_work, 0);
777 } else {
778 spin_unlock(&st->receive_credit_lock);
779 }
780
781 st->first_entry_offset = offset;
782 ksmbd_debug(RDMA,
783 "returning to thread data_read=%d reassembly_data_length=%d first_entry_offset=%d\n",
784 data_read, st->reassembly_data_length,
785 st->first_entry_offset);
786 read_rfc1002_done:
787 return data_read;
788 }
789
790 ksmbd_debug(RDMA, "wait_event on more data\n");
791 rc = wait_event_interruptible(st->wait_reassembly_queue,
792 st->reassembly_data_length >= size ||
793 st->status != SMB_DIRECT_CS_CONNECTED);
794 if (rc)
795 return -EINTR;
796
797 goto again;
798 }
799
smb_direct_post_recv_credits(struct work_struct * work)800 static void smb_direct_post_recv_credits(struct work_struct *work)
801 {
802 struct smb_direct_transport *t = container_of(work,
803 struct smb_direct_transport, post_recv_credits_work.work);
804 struct smb_direct_recvmsg *recvmsg;
805 int receive_credits, credits = 0;
806 int ret;
807 int use_free = 1;
808
809 spin_lock(&t->receive_credit_lock);
810 receive_credits = t->recv_credits;
811 spin_unlock(&t->receive_credit_lock);
812
813 if (receive_credits < t->recv_credit_target) {
814 while (true) {
815 if (use_free)
816 recvmsg = get_free_recvmsg(t);
817 else
818 recvmsg = get_empty_recvmsg(t);
819 if (!recvmsg) {
820 if (use_free) {
821 use_free = 0;
822 continue;
823 } else {
824 break;
825 }
826 }
827
828 recvmsg->type = SMB_DIRECT_MSG_DATA_TRANSFER;
829 recvmsg->first_segment = false;
830
831 ret = smb_direct_post_recv(t, recvmsg);
832 if (ret) {
833 pr_err("Can't post recv: %d\n", ret);
834 put_recvmsg(t, recvmsg);
835 break;
836 }
837 credits++;
838 }
839 }
840
841 spin_lock(&t->receive_credit_lock);
842 t->recv_credits += credits;
843 t->count_avail_recvmsg -= credits;
844 spin_unlock(&t->receive_credit_lock);
845
846 spin_lock(&t->lock_new_recv_credits);
847 t->new_recv_credits += credits;
848 spin_unlock(&t->lock_new_recv_credits);
849
850 if (credits)
851 queue_work(smb_direct_wq, &t->send_immediate_work);
852 }
853
send_done(struct ib_cq * cq,struct ib_wc * wc)854 static void send_done(struct ib_cq *cq, struct ib_wc *wc)
855 {
856 struct smb_direct_sendmsg *sendmsg, *sibling;
857 struct smb_direct_transport *t;
858 struct list_head *pos, *prev, *end;
859
860 sendmsg = container_of(wc->wr_cqe, struct smb_direct_sendmsg, cqe);
861 t = sendmsg->transport;
862
863 ksmbd_debug(RDMA, "Send completed. status='%s (%d)', opcode=%d\n",
864 ib_wc_status_msg(wc->status), wc->status,
865 wc->opcode);
866
867 if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_SEND) {
868 pr_err("Send error. status='%s (%d)', opcode=%d\n",
869 ib_wc_status_msg(wc->status), wc->status,
870 wc->opcode);
871 smb_direct_disconnect_rdma_connection(t);
872 }
873
874 if (atomic_dec_and_test(&t->send_pending))
875 wake_up(&t->wait_send_pending);
876
877 /* iterate and free the list of messages in reverse. the list's head
878 * is invalid.
879 */
880 for (pos = &sendmsg->list, prev = pos->prev, end = sendmsg->list.next;
881 prev != end; pos = prev, prev = prev->prev) {
882 sibling = container_of(pos, struct smb_direct_sendmsg, list);
883 smb_direct_free_sendmsg(t, sibling);
884 }
885
886 sibling = container_of(pos, struct smb_direct_sendmsg, list);
887 smb_direct_free_sendmsg(t, sibling);
888 }
889
manage_credits_prior_sending(struct smb_direct_transport * t)890 static int manage_credits_prior_sending(struct smb_direct_transport *t)
891 {
892 int new_credits;
893
894 spin_lock(&t->lock_new_recv_credits);
895 new_credits = t->new_recv_credits;
896 t->new_recv_credits = 0;
897 spin_unlock(&t->lock_new_recv_credits);
898
899 return new_credits;
900 }
901
smb_direct_post_send(struct smb_direct_transport * t,struct ib_send_wr * wr)902 static int smb_direct_post_send(struct smb_direct_transport *t,
903 struct ib_send_wr *wr)
904 {
905 int ret;
906
907 atomic_inc(&t->send_pending);
908 ret = ib_post_send(t->qp, wr, NULL);
909 if (ret) {
910 pr_err("failed to post send: %d\n", ret);
911 if (atomic_dec_and_test(&t->send_pending))
912 wake_up(&t->wait_send_pending);
913 smb_direct_disconnect_rdma_connection(t);
914 }
915 return ret;
916 }
917
smb_direct_send_ctx_init(struct smb_direct_transport * t,struct smb_direct_send_ctx * send_ctx,bool need_invalidate_rkey,unsigned int remote_key)918 static void smb_direct_send_ctx_init(struct smb_direct_transport *t,
919 struct smb_direct_send_ctx *send_ctx,
920 bool need_invalidate_rkey,
921 unsigned int remote_key)
922 {
923 INIT_LIST_HEAD(&send_ctx->msg_list);
924 send_ctx->wr_cnt = 0;
925 send_ctx->need_invalidate_rkey = need_invalidate_rkey;
926 send_ctx->remote_key = remote_key;
927 }
928
smb_direct_flush_send_list(struct smb_direct_transport * t,struct smb_direct_send_ctx * send_ctx,bool is_last)929 static int smb_direct_flush_send_list(struct smb_direct_transport *t,
930 struct smb_direct_send_ctx *send_ctx,
931 bool is_last)
932 {
933 struct smb_direct_sendmsg *first, *last;
934 int ret;
935
936 if (list_empty(&send_ctx->msg_list))
937 return 0;
938
939 first = list_first_entry(&send_ctx->msg_list,
940 struct smb_direct_sendmsg,
941 list);
942 last = list_last_entry(&send_ctx->msg_list,
943 struct smb_direct_sendmsg,
944 list);
945
946 last->wr.send_flags = IB_SEND_SIGNALED;
947 last->wr.wr_cqe = &last->cqe;
948 if (is_last && send_ctx->need_invalidate_rkey) {
949 last->wr.opcode = IB_WR_SEND_WITH_INV;
950 last->wr.ex.invalidate_rkey = send_ctx->remote_key;
951 }
952
953 ret = smb_direct_post_send(t, &first->wr);
954 if (!ret) {
955 smb_direct_send_ctx_init(t, send_ctx,
956 send_ctx->need_invalidate_rkey,
957 send_ctx->remote_key);
958 } else {
959 atomic_add(send_ctx->wr_cnt, &t->send_credits);
960 wake_up(&t->wait_send_credits);
961 list_for_each_entry_safe(first, last, &send_ctx->msg_list,
962 list) {
963 smb_direct_free_sendmsg(t, first);
964 }
965 }
966 return ret;
967 }
968
wait_for_credits(struct smb_direct_transport * t,wait_queue_head_t * waitq,atomic_t * total_credits,int needed)969 static int wait_for_credits(struct smb_direct_transport *t,
970 wait_queue_head_t *waitq, atomic_t *total_credits,
971 int needed)
972 {
973 int ret;
974
975 do {
976 if (atomic_sub_return(needed, total_credits) >= 0)
977 return 0;
978
979 atomic_add(needed, total_credits);
980 ret = wait_event_interruptible(*waitq,
981 atomic_read(total_credits) >= needed ||
982 t->status != SMB_DIRECT_CS_CONNECTED);
983
984 if (t->status != SMB_DIRECT_CS_CONNECTED)
985 return -ENOTCONN;
986 else if (ret < 0)
987 return ret;
988 } while (true);
989 }
990
wait_for_send_credits(struct smb_direct_transport * t,struct smb_direct_send_ctx * send_ctx)991 static int wait_for_send_credits(struct smb_direct_transport *t,
992 struct smb_direct_send_ctx *send_ctx)
993 {
994 int ret;
995
996 if (send_ctx &&
997 (send_ctx->wr_cnt >= 16 || atomic_read(&t->send_credits) <= 1)) {
998 ret = smb_direct_flush_send_list(t, send_ctx, false);
999 if (ret)
1000 return ret;
1001 }
1002
1003 return wait_for_credits(t, &t->wait_send_credits, &t->send_credits, 1);
1004 }
1005
wait_for_rw_credits(struct smb_direct_transport * t,int credits)1006 static int wait_for_rw_credits(struct smb_direct_transport *t, int credits)
1007 {
1008 return wait_for_credits(t, &t->wait_rw_credits, &t->rw_credits, credits);
1009 }
1010
calc_rw_credits(struct smb_direct_transport * t,char * buf,unsigned int len)1011 static int calc_rw_credits(struct smb_direct_transport *t,
1012 char *buf, unsigned int len)
1013 {
1014 return DIV_ROUND_UP(get_buf_page_count(buf, len),
1015 t->pages_per_rw_credit);
1016 }
1017
smb_direct_create_header(struct smb_direct_transport * t,int size,int remaining_data_length,struct smb_direct_sendmsg ** sendmsg_out)1018 static int smb_direct_create_header(struct smb_direct_transport *t,
1019 int size, int remaining_data_length,
1020 struct smb_direct_sendmsg **sendmsg_out)
1021 {
1022 struct smb_direct_sendmsg *sendmsg;
1023 struct smb_direct_data_transfer *packet;
1024 int header_length;
1025 int ret;
1026
1027 sendmsg = smb_direct_alloc_sendmsg(t);
1028 if (IS_ERR(sendmsg))
1029 return PTR_ERR(sendmsg);
1030
1031 /* Fill in the packet header */
1032 packet = (struct smb_direct_data_transfer *)sendmsg->packet;
1033 packet->credits_requested = cpu_to_le16(t->send_credit_target);
1034 packet->credits_granted = cpu_to_le16(manage_credits_prior_sending(t));
1035
1036 packet->flags = 0;
1037 packet->reserved = 0;
1038 if (!size)
1039 packet->data_offset = 0;
1040 else
1041 packet->data_offset = cpu_to_le32(24);
1042 packet->data_length = cpu_to_le32(size);
1043 packet->remaining_data_length = cpu_to_le32(remaining_data_length);
1044 packet->padding = 0;
1045
1046 ksmbd_debug(RDMA,
1047 "credits_requested=%d credits_granted=%d data_offset=%d data_length=%d remaining_data_length=%d\n",
1048 le16_to_cpu(packet->credits_requested),
1049 le16_to_cpu(packet->credits_granted),
1050 le32_to_cpu(packet->data_offset),
1051 le32_to_cpu(packet->data_length),
1052 le32_to_cpu(packet->remaining_data_length));
1053
1054 /* Map the packet to DMA */
1055 header_length = sizeof(struct smb_direct_data_transfer);
1056 /* If this is a packet without payload, don't send padding */
1057 if (!size)
1058 header_length =
1059 offsetof(struct smb_direct_data_transfer, padding);
1060
1061 sendmsg->sge[0].addr = ib_dma_map_single(t->cm_id->device,
1062 (void *)packet,
1063 header_length,
1064 DMA_TO_DEVICE);
1065 ret = ib_dma_mapping_error(t->cm_id->device, sendmsg->sge[0].addr);
1066 if (ret) {
1067 smb_direct_free_sendmsg(t, sendmsg);
1068 return ret;
1069 }
1070
1071 sendmsg->num_sge = 1;
1072 sendmsg->sge[0].length = header_length;
1073 sendmsg->sge[0].lkey = t->pd->local_dma_lkey;
1074
1075 *sendmsg_out = sendmsg;
1076 return 0;
1077 }
1078
get_sg_list(void * buf,int size,struct scatterlist * sg_list,int nentries)1079 static int get_sg_list(void *buf, int size, struct scatterlist *sg_list, int nentries)
1080 {
1081 bool high = is_vmalloc_addr(buf);
1082 struct page *page;
1083 int offset, len;
1084 int i = 0;
1085
1086 if (size <= 0 || nentries < get_buf_page_count(buf, size))
1087 return -EINVAL;
1088
1089 offset = offset_in_page(buf);
1090 buf -= offset;
1091 while (size > 0) {
1092 len = min_t(int, PAGE_SIZE - offset, size);
1093 if (high)
1094 page = vmalloc_to_page(buf);
1095 else
1096 page = kmap_to_page(buf);
1097
1098 if (!sg_list)
1099 return -EINVAL;
1100 sg_set_page(sg_list, page, len, offset);
1101 sg_list = sg_next(sg_list);
1102
1103 buf += PAGE_SIZE;
1104 size -= len;
1105 offset = 0;
1106 i++;
1107 }
1108 return i;
1109 }
1110
get_mapped_sg_list(struct ib_device * device,void * buf,int size,struct scatterlist * sg_list,int nentries,enum dma_data_direction dir)1111 static int get_mapped_sg_list(struct ib_device *device, void *buf, int size,
1112 struct scatterlist *sg_list, int nentries,
1113 enum dma_data_direction dir)
1114 {
1115 int npages;
1116
1117 npages = get_sg_list(buf, size, sg_list, nentries);
1118 if (npages < 0)
1119 return -EINVAL;
1120 return ib_dma_map_sg(device, sg_list, npages, dir);
1121 }
1122
post_sendmsg(struct smb_direct_transport * t,struct smb_direct_send_ctx * send_ctx,struct smb_direct_sendmsg * msg)1123 static int post_sendmsg(struct smb_direct_transport *t,
1124 struct smb_direct_send_ctx *send_ctx,
1125 struct smb_direct_sendmsg *msg)
1126 {
1127 int i;
1128
1129 for (i = 0; i < msg->num_sge; i++)
1130 ib_dma_sync_single_for_device(t->cm_id->device,
1131 msg->sge[i].addr, msg->sge[i].length,
1132 DMA_TO_DEVICE);
1133
1134 msg->cqe.done = send_done;
1135 msg->wr.opcode = IB_WR_SEND;
1136 msg->wr.sg_list = &msg->sge[0];
1137 msg->wr.num_sge = msg->num_sge;
1138 msg->wr.next = NULL;
1139
1140 if (send_ctx) {
1141 msg->wr.wr_cqe = NULL;
1142 msg->wr.send_flags = 0;
1143 if (!list_empty(&send_ctx->msg_list)) {
1144 struct smb_direct_sendmsg *last;
1145
1146 last = list_last_entry(&send_ctx->msg_list,
1147 struct smb_direct_sendmsg,
1148 list);
1149 last->wr.next = &msg->wr;
1150 }
1151 list_add_tail(&msg->list, &send_ctx->msg_list);
1152 send_ctx->wr_cnt++;
1153 return 0;
1154 }
1155
1156 msg->wr.wr_cqe = &msg->cqe;
1157 msg->wr.send_flags = IB_SEND_SIGNALED;
1158 return smb_direct_post_send(t, &msg->wr);
1159 }
1160
smb_direct_post_send_data(struct smb_direct_transport * t,struct smb_direct_send_ctx * send_ctx,struct kvec * iov,int niov,int remaining_data_length)1161 static int smb_direct_post_send_data(struct smb_direct_transport *t,
1162 struct smb_direct_send_ctx *send_ctx,
1163 struct kvec *iov, int niov,
1164 int remaining_data_length)
1165 {
1166 int i, j, ret;
1167 struct smb_direct_sendmsg *msg;
1168 int data_length;
1169 struct scatterlist sg[SMB_DIRECT_MAX_SEND_SGES - 1];
1170
1171 ret = wait_for_send_credits(t, send_ctx);
1172 if (ret)
1173 return ret;
1174
1175 data_length = 0;
1176 for (i = 0; i < niov; i++)
1177 data_length += iov[i].iov_len;
1178
1179 ret = smb_direct_create_header(t, data_length, remaining_data_length,
1180 &msg);
1181 if (ret) {
1182 atomic_inc(&t->send_credits);
1183 return ret;
1184 }
1185
1186 for (i = 0; i < niov; i++) {
1187 struct ib_sge *sge;
1188 int sg_cnt;
1189
1190 sg_init_table(sg, SMB_DIRECT_MAX_SEND_SGES - 1);
1191 sg_cnt = get_mapped_sg_list(t->cm_id->device,
1192 iov[i].iov_base, iov[i].iov_len,
1193 sg, SMB_DIRECT_MAX_SEND_SGES - 1,
1194 DMA_TO_DEVICE);
1195 if (sg_cnt <= 0) {
1196 pr_err("failed to map buffer\n");
1197 ret = -ENOMEM;
1198 goto err;
1199 } else if (sg_cnt + msg->num_sge > SMB_DIRECT_MAX_SEND_SGES) {
1200 pr_err("buffer not fitted into sges\n");
1201 ret = -E2BIG;
1202 ib_dma_unmap_sg(t->cm_id->device, sg, sg_cnt,
1203 DMA_TO_DEVICE);
1204 goto err;
1205 }
1206
1207 for (j = 0; j < sg_cnt; j++) {
1208 sge = &msg->sge[msg->num_sge];
1209 sge->addr = sg_dma_address(&sg[j]);
1210 sge->length = sg_dma_len(&sg[j]);
1211 sge->lkey = t->pd->local_dma_lkey;
1212 msg->num_sge++;
1213 }
1214 }
1215
1216 ret = post_sendmsg(t, send_ctx, msg);
1217 if (ret)
1218 goto err;
1219 return 0;
1220 err:
1221 smb_direct_free_sendmsg(t, msg);
1222 atomic_inc(&t->send_credits);
1223 return ret;
1224 }
1225
smb_direct_writev(struct ksmbd_transport * t,struct kvec * iov,int niovs,int buflen,bool need_invalidate,unsigned int remote_key)1226 static int smb_direct_writev(struct ksmbd_transport *t,
1227 struct kvec *iov, int niovs, int buflen,
1228 bool need_invalidate, unsigned int remote_key)
1229 {
1230 struct smb_direct_transport *st = smb_trans_direct_transfort(t);
1231 int remaining_data_length;
1232 int start, i, j;
1233 int max_iov_size = st->max_send_size -
1234 sizeof(struct smb_direct_data_transfer);
1235 int ret;
1236 struct kvec vec;
1237 struct smb_direct_send_ctx send_ctx;
1238
1239 if (st->status != SMB_DIRECT_CS_CONNECTED)
1240 return -ENOTCONN;
1241
1242 //FIXME: skip RFC1002 header..
1243 buflen -= 4;
1244
1245 remaining_data_length = buflen;
1246 ksmbd_debug(RDMA, "Sending smb (RDMA): smb_len=%u\n", buflen);
1247
1248 smb_direct_send_ctx_init(st, &send_ctx, need_invalidate, remote_key);
1249 start = i = 1;
1250 buflen = 0;
1251 while (true) {
1252 buflen += iov[i].iov_len;
1253 if (buflen > max_iov_size) {
1254 if (i > start) {
1255 remaining_data_length -=
1256 (buflen - iov[i].iov_len);
1257 ret = smb_direct_post_send_data(st, &send_ctx,
1258 &iov[start], i - start,
1259 remaining_data_length);
1260 if (ret)
1261 goto done;
1262 } else {
1263 /* iov[start] is too big, break it */
1264 int nvec = (buflen + max_iov_size - 1) /
1265 max_iov_size;
1266
1267 for (j = 0; j < nvec; j++) {
1268 vec.iov_base =
1269 (char *)iov[start].iov_base +
1270 j * max_iov_size;
1271 vec.iov_len =
1272 min_t(int, max_iov_size,
1273 buflen - max_iov_size * j);
1274 remaining_data_length -= vec.iov_len;
1275 ret = smb_direct_post_send_data(st, &send_ctx, &vec, 1,
1276 remaining_data_length);
1277 if (ret)
1278 goto done;
1279 }
1280 i++;
1281 if (i == niovs)
1282 break;
1283 }
1284 start = i;
1285 buflen = 0;
1286 } else {
1287 i++;
1288 if (i == niovs) {
1289 /* send out all remaining vecs */
1290 remaining_data_length -= buflen;
1291 ret = smb_direct_post_send_data(st, &send_ctx,
1292 &iov[start], i - start,
1293 remaining_data_length);
1294 if (ret)
1295 goto done;
1296 break;
1297 }
1298 }
1299 }
1300
1301 done:
1302 ret = smb_direct_flush_send_list(st, &send_ctx, true);
1303
1304 /*
1305 * As an optimization, we don't wait for individual I/O to finish
1306 * before sending the next one.
1307 * Send them all and wait for pending send count to get to 0
1308 * that means all the I/Os have been out and we are good to return
1309 */
1310
1311 wait_event(st->wait_send_pending,
1312 atomic_read(&st->send_pending) == 0);
1313 return ret;
1314 }
1315
smb_direct_free_rdma_rw_msg(struct smb_direct_transport * t,struct smb_direct_rdma_rw_msg * msg,enum dma_data_direction dir)1316 static void smb_direct_free_rdma_rw_msg(struct smb_direct_transport *t,
1317 struct smb_direct_rdma_rw_msg *msg,
1318 enum dma_data_direction dir)
1319 {
1320 rdma_rw_ctx_destroy(&msg->rw_ctx, t->qp, t->qp->port,
1321 msg->sgt.sgl, msg->sgt.nents, dir);
1322 sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE);
1323 kfree(msg);
1324 }
1325
read_write_done(struct ib_cq * cq,struct ib_wc * wc,enum dma_data_direction dir)1326 static void read_write_done(struct ib_cq *cq, struct ib_wc *wc,
1327 enum dma_data_direction dir)
1328 {
1329 struct smb_direct_rdma_rw_msg *msg = container_of(wc->wr_cqe,
1330 struct smb_direct_rdma_rw_msg, cqe);
1331 struct smb_direct_transport *t = msg->t;
1332
1333 if (wc->status != IB_WC_SUCCESS) {
1334 msg->status = -EIO;
1335 pr_err("read/write error. opcode = %d, status = %s(%d)\n",
1336 wc->opcode, ib_wc_status_msg(wc->status), wc->status);
1337 if (wc->status != IB_WC_WR_FLUSH_ERR)
1338 smb_direct_disconnect_rdma_connection(t);
1339 }
1340
1341 complete(msg->completion);
1342 }
1343
read_done(struct ib_cq * cq,struct ib_wc * wc)1344 static void read_done(struct ib_cq *cq, struct ib_wc *wc)
1345 {
1346 read_write_done(cq, wc, DMA_FROM_DEVICE);
1347 }
1348
write_done(struct ib_cq * cq,struct ib_wc * wc)1349 static void write_done(struct ib_cq *cq, struct ib_wc *wc)
1350 {
1351 read_write_done(cq, wc, DMA_TO_DEVICE);
1352 }
1353
smb_direct_rdma_xmit(struct smb_direct_transport * t,void * buf,int buf_len,struct smb2_buffer_desc_v1 * desc,unsigned int desc_len,bool is_read)1354 static int smb_direct_rdma_xmit(struct smb_direct_transport *t,
1355 void *buf, int buf_len,
1356 struct smb2_buffer_desc_v1 *desc,
1357 unsigned int desc_len,
1358 bool is_read)
1359 {
1360 struct smb_direct_rdma_rw_msg *msg, *next_msg;
1361 int i, ret;
1362 DECLARE_COMPLETION_ONSTACK(completion);
1363 struct ib_send_wr *first_wr;
1364 LIST_HEAD(msg_list);
1365 char *desc_buf;
1366 int credits_needed;
1367 unsigned int desc_buf_len, desc_num = 0;
1368
1369 if (t->status != SMB_DIRECT_CS_CONNECTED)
1370 return -ENOTCONN;
1371
1372 if (buf_len > t->max_rdma_rw_size)
1373 return -EINVAL;
1374
1375 /* calculate needed credits */
1376 credits_needed = 0;
1377 desc_buf = buf;
1378 for (i = 0; i < desc_len / sizeof(*desc); i++) {
1379 if (!buf_len)
1380 break;
1381
1382 desc_buf_len = le32_to_cpu(desc[i].length);
1383 if (!desc_buf_len)
1384 return -EINVAL;
1385
1386 if (desc_buf_len > buf_len) {
1387 desc_buf_len = buf_len;
1388 desc[i].length = cpu_to_le32(desc_buf_len);
1389 buf_len = 0;
1390 }
1391
1392 credits_needed += calc_rw_credits(t, desc_buf, desc_buf_len);
1393 desc_buf += desc_buf_len;
1394 buf_len -= desc_buf_len;
1395 desc_num++;
1396 }
1397
1398 ksmbd_debug(RDMA, "RDMA %s, len %#x, needed credits %#x\n",
1399 is_read ? "read" : "write", buf_len, credits_needed);
1400
1401 ret = wait_for_rw_credits(t, credits_needed);
1402 if (ret < 0)
1403 return ret;
1404
1405 /* build rdma_rw_ctx for each descriptor */
1406 desc_buf = buf;
1407 for (i = 0; i < desc_num; i++) {
1408 msg = kzalloc(struct_size(msg, sg_list, SG_CHUNK_SIZE),
1409 GFP_KERNEL);
1410 if (!msg) {
1411 ret = -ENOMEM;
1412 goto out;
1413 }
1414
1415 desc_buf_len = le32_to_cpu(desc[i].length);
1416
1417 msg->t = t;
1418 msg->cqe.done = is_read ? read_done : write_done;
1419 msg->completion = &completion;
1420
1421 msg->sgt.sgl = &msg->sg_list[0];
1422 ret = sg_alloc_table_chained(&msg->sgt,
1423 get_buf_page_count(desc_buf, desc_buf_len),
1424 msg->sg_list, SG_CHUNK_SIZE);
1425 if (ret) {
1426 kfree(msg);
1427 ret = -ENOMEM;
1428 goto out;
1429 }
1430
1431 ret = get_sg_list(desc_buf, desc_buf_len,
1432 msg->sgt.sgl, msg->sgt.orig_nents);
1433 if (ret < 0) {
1434 sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE);
1435 kfree(msg);
1436 goto out;
1437 }
1438
1439 ret = rdma_rw_ctx_init(&msg->rw_ctx, t->qp, t->qp->port,
1440 msg->sgt.sgl,
1441 get_buf_page_count(desc_buf, desc_buf_len),
1442 0,
1443 le64_to_cpu(desc[i].offset),
1444 le32_to_cpu(desc[i].token),
1445 is_read ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
1446 if (ret < 0) {
1447 pr_err("failed to init rdma_rw_ctx: %d\n", ret);
1448 sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE);
1449 kfree(msg);
1450 goto out;
1451 }
1452
1453 list_add_tail(&msg->list, &msg_list);
1454 desc_buf += desc_buf_len;
1455 }
1456
1457 /* concatenate work requests of rdma_rw_ctxs */
1458 first_wr = NULL;
1459 list_for_each_entry_reverse(msg, &msg_list, list) {
1460 first_wr = rdma_rw_ctx_wrs(&msg->rw_ctx, t->qp, t->qp->port,
1461 &msg->cqe, first_wr);
1462 }
1463
1464 ret = ib_post_send(t->qp, first_wr, NULL);
1465 if (ret) {
1466 pr_err("failed to post send wr for RDMA R/W: %d\n", ret);
1467 goto out;
1468 }
1469
1470 msg = list_last_entry(&msg_list, struct smb_direct_rdma_rw_msg, list);
1471 wait_for_completion(&completion);
1472 ret = msg->status;
1473 out:
1474 list_for_each_entry_safe(msg, next_msg, &msg_list, list) {
1475 list_del(&msg->list);
1476 smb_direct_free_rdma_rw_msg(t, msg,
1477 is_read ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
1478 }
1479 atomic_add(credits_needed, &t->rw_credits);
1480 wake_up(&t->wait_rw_credits);
1481 return ret;
1482 }
1483
smb_direct_rdma_write(struct ksmbd_transport * t,void * buf,unsigned int buflen,struct smb2_buffer_desc_v1 * desc,unsigned int desc_len)1484 static int smb_direct_rdma_write(struct ksmbd_transport *t,
1485 void *buf, unsigned int buflen,
1486 struct smb2_buffer_desc_v1 *desc,
1487 unsigned int desc_len)
1488 {
1489 return smb_direct_rdma_xmit(smb_trans_direct_transfort(t), buf, buflen,
1490 desc, desc_len, false);
1491 }
1492
smb_direct_rdma_read(struct ksmbd_transport * t,void * buf,unsigned int buflen,struct smb2_buffer_desc_v1 * desc,unsigned int desc_len)1493 static int smb_direct_rdma_read(struct ksmbd_transport *t,
1494 void *buf, unsigned int buflen,
1495 struct smb2_buffer_desc_v1 *desc,
1496 unsigned int desc_len)
1497 {
1498 return smb_direct_rdma_xmit(smb_trans_direct_transfort(t), buf, buflen,
1499 desc, desc_len, true);
1500 }
1501
smb_direct_disconnect(struct ksmbd_transport * t)1502 static void smb_direct_disconnect(struct ksmbd_transport *t)
1503 {
1504 struct smb_direct_transport *st = smb_trans_direct_transfort(t);
1505
1506 ksmbd_debug(RDMA, "Disconnecting cm_id=%p\n", st->cm_id);
1507
1508 smb_direct_disconnect_rdma_work(&st->disconnect_work);
1509 wait_event_interruptible(st->wait_status,
1510 st->status == SMB_DIRECT_CS_DISCONNECTED);
1511 free_transport(st);
1512 }
1513
smb_direct_shutdown(struct ksmbd_transport * t)1514 static void smb_direct_shutdown(struct ksmbd_transport *t)
1515 {
1516 struct smb_direct_transport *st = smb_trans_direct_transfort(t);
1517
1518 ksmbd_debug(RDMA, "smb-direct shutdown cm_id=%p\n", st->cm_id);
1519
1520 smb_direct_disconnect_rdma_work(&st->disconnect_work);
1521 }
1522
smb_direct_cm_handler(struct rdma_cm_id * cm_id,struct rdma_cm_event * event)1523 static int smb_direct_cm_handler(struct rdma_cm_id *cm_id,
1524 struct rdma_cm_event *event)
1525 {
1526 struct smb_direct_transport *t = cm_id->context;
1527
1528 ksmbd_debug(RDMA, "RDMA CM event. cm_id=%p event=%s (%d)\n",
1529 cm_id, rdma_event_msg(event->event), event->event);
1530
1531 switch (event->event) {
1532 case RDMA_CM_EVENT_ESTABLISHED: {
1533 t->status = SMB_DIRECT_CS_CONNECTED;
1534 wake_up_interruptible(&t->wait_status);
1535 break;
1536 }
1537 case RDMA_CM_EVENT_DEVICE_REMOVAL:
1538 case RDMA_CM_EVENT_DISCONNECTED: {
1539 ib_drain_qp(t->qp);
1540
1541 t->status = SMB_DIRECT_CS_DISCONNECTED;
1542 wake_up_interruptible(&t->wait_status);
1543 wake_up_interruptible(&t->wait_reassembly_queue);
1544 wake_up(&t->wait_send_credits);
1545 break;
1546 }
1547 case RDMA_CM_EVENT_CONNECT_ERROR: {
1548 t->status = SMB_DIRECT_CS_DISCONNECTED;
1549 wake_up_interruptible(&t->wait_status);
1550 break;
1551 }
1552 default:
1553 pr_err("Unexpected RDMA CM event. cm_id=%p, event=%s (%d)\n",
1554 cm_id, rdma_event_msg(event->event),
1555 event->event);
1556 break;
1557 }
1558 return 0;
1559 }
1560
smb_direct_qpair_handler(struct ib_event * event,void * context)1561 static void smb_direct_qpair_handler(struct ib_event *event, void *context)
1562 {
1563 struct smb_direct_transport *t = context;
1564
1565 ksmbd_debug(RDMA, "Received QP event. cm_id=%p, event=%s (%d)\n",
1566 t->cm_id, ib_event_msg(event->event), event->event);
1567
1568 switch (event->event) {
1569 case IB_EVENT_CQ_ERR:
1570 case IB_EVENT_QP_FATAL:
1571 smb_direct_disconnect_rdma_connection(t);
1572 break;
1573 default:
1574 break;
1575 }
1576 }
1577
smb_direct_send_negotiate_response(struct smb_direct_transport * t,int failed)1578 static int smb_direct_send_negotiate_response(struct smb_direct_transport *t,
1579 int failed)
1580 {
1581 struct smb_direct_sendmsg *sendmsg;
1582 struct smb_direct_negotiate_resp *resp;
1583 int ret;
1584
1585 sendmsg = smb_direct_alloc_sendmsg(t);
1586 if (IS_ERR(sendmsg))
1587 return -ENOMEM;
1588
1589 resp = (struct smb_direct_negotiate_resp *)sendmsg->packet;
1590 if (failed) {
1591 memset(resp, 0, sizeof(*resp));
1592 resp->min_version = cpu_to_le16(0x0100);
1593 resp->max_version = cpu_to_le16(0x0100);
1594 resp->status = STATUS_NOT_SUPPORTED;
1595 } else {
1596 resp->status = STATUS_SUCCESS;
1597 resp->min_version = SMB_DIRECT_VERSION_LE;
1598 resp->max_version = SMB_DIRECT_VERSION_LE;
1599 resp->negotiated_version = SMB_DIRECT_VERSION_LE;
1600 resp->reserved = 0;
1601 resp->credits_requested =
1602 cpu_to_le16(t->send_credit_target);
1603 resp->credits_granted = cpu_to_le16(manage_credits_prior_sending(t));
1604 resp->max_readwrite_size = cpu_to_le32(t->max_rdma_rw_size);
1605 resp->preferred_send_size = cpu_to_le32(t->max_send_size);
1606 resp->max_receive_size = cpu_to_le32(t->max_recv_size);
1607 resp->max_fragmented_size =
1608 cpu_to_le32(t->max_fragmented_recv_size);
1609 }
1610
1611 sendmsg->sge[0].addr = ib_dma_map_single(t->cm_id->device,
1612 (void *)resp, sizeof(*resp),
1613 DMA_TO_DEVICE);
1614 ret = ib_dma_mapping_error(t->cm_id->device, sendmsg->sge[0].addr);
1615 if (ret) {
1616 smb_direct_free_sendmsg(t, sendmsg);
1617 return ret;
1618 }
1619
1620 sendmsg->num_sge = 1;
1621 sendmsg->sge[0].length = sizeof(*resp);
1622 sendmsg->sge[0].lkey = t->pd->local_dma_lkey;
1623
1624 ret = post_sendmsg(t, NULL, sendmsg);
1625 if (ret) {
1626 smb_direct_free_sendmsg(t, sendmsg);
1627 return ret;
1628 }
1629
1630 wait_event(t->wait_send_pending,
1631 atomic_read(&t->send_pending) == 0);
1632 return 0;
1633 }
1634
smb_direct_accept_client(struct smb_direct_transport * t)1635 static int smb_direct_accept_client(struct smb_direct_transport *t)
1636 {
1637 struct rdma_conn_param conn_param;
1638 struct ib_port_immutable port_immutable;
1639 u32 ird_ord_hdr[2];
1640 int ret;
1641
1642 memset(&conn_param, 0, sizeof(conn_param));
1643 conn_param.initiator_depth = min_t(u8, t->cm_id->device->attrs.max_qp_rd_atom,
1644 SMB_DIRECT_CM_INITIATOR_DEPTH);
1645 conn_param.responder_resources = 0;
1646
1647 t->cm_id->device->ops.get_port_immutable(t->cm_id->device,
1648 t->cm_id->port_num,
1649 &port_immutable);
1650 if (port_immutable.core_cap_flags & RDMA_CORE_PORT_IWARP) {
1651 ird_ord_hdr[0] = conn_param.responder_resources;
1652 ird_ord_hdr[1] = 1;
1653 conn_param.private_data = ird_ord_hdr;
1654 conn_param.private_data_len = sizeof(ird_ord_hdr);
1655 } else {
1656 conn_param.private_data = NULL;
1657 conn_param.private_data_len = 0;
1658 }
1659 conn_param.retry_count = SMB_DIRECT_CM_RETRY;
1660 conn_param.rnr_retry_count = SMB_DIRECT_CM_RNR_RETRY;
1661 conn_param.flow_control = 0;
1662
1663 ret = rdma_accept(t->cm_id, &conn_param);
1664 if (ret) {
1665 pr_err("error at rdma_accept: %d\n", ret);
1666 return ret;
1667 }
1668 return 0;
1669 }
1670
smb_direct_prepare_negotiation(struct smb_direct_transport * t)1671 static int smb_direct_prepare_negotiation(struct smb_direct_transport *t)
1672 {
1673 int ret;
1674 struct smb_direct_recvmsg *recvmsg;
1675
1676 recvmsg = get_free_recvmsg(t);
1677 if (!recvmsg)
1678 return -ENOMEM;
1679 recvmsg->type = SMB_DIRECT_MSG_NEGOTIATE_REQ;
1680
1681 ret = smb_direct_post_recv(t, recvmsg);
1682 if (ret) {
1683 pr_err("Can't post recv: %d\n", ret);
1684 goto out_err;
1685 }
1686
1687 t->negotiation_requested = false;
1688 ret = smb_direct_accept_client(t);
1689 if (ret) {
1690 pr_err("Can't accept client\n");
1691 goto out_err;
1692 }
1693
1694 smb_direct_post_recv_credits(&t->post_recv_credits_work.work);
1695 return 0;
1696 out_err:
1697 put_recvmsg(t, recvmsg);
1698 return ret;
1699 }
1700
smb_direct_get_max_fr_pages(struct smb_direct_transport * t)1701 static unsigned int smb_direct_get_max_fr_pages(struct smb_direct_transport *t)
1702 {
1703 return min_t(unsigned int,
1704 t->cm_id->device->attrs.max_fast_reg_page_list_len,
1705 256);
1706 }
1707
smb_direct_init_params(struct smb_direct_transport * t,struct ib_qp_cap * cap)1708 static int smb_direct_init_params(struct smb_direct_transport *t,
1709 struct ib_qp_cap *cap)
1710 {
1711 struct ib_device *device = t->cm_id->device;
1712 int max_send_sges, max_rw_wrs, max_send_wrs;
1713 unsigned int max_sge_per_wr, wrs_per_credit;
1714
1715 /* need 3 more sge. because a SMB_DIRECT header, SMB2 header,
1716 * SMB2 response could be mapped.
1717 */
1718 t->max_send_size = smb_direct_max_send_size;
1719 max_send_sges = DIV_ROUND_UP(t->max_send_size, PAGE_SIZE) + 3;
1720 if (max_send_sges > SMB_DIRECT_MAX_SEND_SGES) {
1721 pr_err("max_send_size %d is too large\n", t->max_send_size);
1722 return -EINVAL;
1723 }
1724
1725 /* Calculate the number of work requests for RDMA R/W.
1726 * The maximum number of pages which can be registered
1727 * with one Memory region can be transferred with one
1728 * R/W credit. And at least 4 work requests for each credit
1729 * are needed for MR registration, RDMA R/W, local & remote
1730 * MR invalidation.
1731 */
1732 t->max_rdma_rw_size = smb_direct_max_read_write_size;
1733 t->pages_per_rw_credit = smb_direct_get_max_fr_pages(t);
1734 t->max_rw_credits = DIV_ROUND_UP(t->max_rdma_rw_size,
1735 (t->pages_per_rw_credit - 1) *
1736 PAGE_SIZE);
1737
1738 max_sge_per_wr = min_t(unsigned int, device->attrs.max_send_sge,
1739 device->attrs.max_sge_rd);
1740 max_sge_per_wr = max_t(unsigned int, max_sge_per_wr,
1741 max_send_sges);
1742 wrs_per_credit = max_t(unsigned int, 4,
1743 DIV_ROUND_UP(t->pages_per_rw_credit,
1744 max_sge_per_wr) + 1);
1745 max_rw_wrs = t->max_rw_credits * wrs_per_credit;
1746
1747 max_send_wrs = smb_direct_send_credit_target + max_rw_wrs;
1748 if (max_send_wrs > device->attrs.max_cqe ||
1749 max_send_wrs > device->attrs.max_qp_wr) {
1750 pr_err("consider lowering send_credit_target = %d\n",
1751 smb_direct_send_credit_target);
1752 pr_err("Possible CQE overrun, device reporting max_cqe %d max_qp_wr %d\n",
1753 device->attrs.max_cqe, device->attrs.max_qp_wr);
1754 return -EINVAL;
1755 }
1756
1757 if (smb_direct_receive_credit_max > device->attrs.max_cqe ||
1758 smb_direct_receive_credit_max > device->attrs.max_qp_wr) {
1759 pr_err("consider lowering receive_credit_max = %d\n",
1760 smb_direct_receive_credit_max);
1761 pr_err("Possible CQE overrun, device reporting max_cpe %d max_qp_wr %d\n",
1762 device->attrs.max_cqe, device->attrs.max_qp_wr);
1763 return -EINVAL;
1764 }
1765
1766 if (device->attrs.max_recv_sge < SMB_DIRECT_MAX_RECV_SGES) {
1767 pr_err("warning: device max_recv_sge = %d too small\n",
1768 device->attrs.max_recv_sge);
1769 return -EINVAL;
1770 }
1771
1772 t->recv_credits = 0;
1773 t->count_avail_recvmsg = 0;
1774
1775 t->recv_credit_max = smb_direct_receive_credit_max;
1776 t->recv_credit_target = 10;
1777 t->new_recv_credits = 0;
1778
1779 t->send_credit_target = smb_direct_send_credit_target;
1780 atomic_set(&t->send_credits, 0);
1781 atomic_set(&t->rw_credits, t->max_rw_credits);
1782
1783 t->max_send_size = smb_direct_max_send_size;
1784 t->max_recv_size = smb_direct_max_receive_size;
1785 t->max_fragmented_recv_size = smb_direct_max_fragmented_recv_size;
1786
1787 cap->max_send_wr = max_send_wrs;
1788 cap->max_recv_wr = t->recv_credit_max;
1789 cap->max_send_sge = max_sge_per_wr;
1790 cap->max_recv_sge = SMB_DIRECT_MAX_RECV_SGES;
1791 cap->max_inline_data = 0;
1792 cap->max_rdma_ctxs = t->max_rw_credits;
1793 return 0;
1794 }
1795
smb_direct_destroy_pools(struct smb_direct_transport * t)1796 static void smb_direct_destroy_pools(struct smb_direct_transport *t)
1797 {
1798 struct smb_direct_recvmsg *recvmsg;
1799
1800 while ((recvmsg = get_free_recvmsg(t)))
1801 mempool_free(recvmsg, t->recvmsg_mempool);
1802 while ((recvmsg = get_empty_recvmsg(t)))
1803 mempool_free(recvmsg, t->recvmsg_mempool);
1804
1805 mempool_destroy(t->recvmsg_mempool);
1806 t->recvmsg_mempool = NULL;
1807
1808 kmem_cache_destroy(t->recvmsg_cache);
1809 t->recvmsg_cache = NULL;
1810
1811 mempool_destroy(t->sendmsg_mempool);
1812 t->sendmsg_mempool = NULL;
1813
1814 kmem_cache_destroy(t->sendmsg_cache);
1815 t->sendmsg_cache = NULL;
1816 }
1817
smb_direct_create_pools(struct smb_direct_transport * t)1818 static int smb_direct_create_pools(struct smb_direct_transport *t)
1819 {
1820 char name[80];
1821 int i;
1822 struct smb_direct_recvmsg *recvmsg;
1823
1824 snprintf(name, sizeof(name), "smb_direct_rqst_pool_%p", t);
1825 t->sendmsg_cache = kmem_cache_create(name,
1826 sizeof(struct smb_direct_sendmsg) +
1827 sizeof(struct smb_direct_negotiate_resp),
1828 0, SLAB_HWCACHE_ALIGN, NULL);
1829 if (!t->sendmsg_cache)
1830 return -ENOMEM;
1831
1832 t->sendmsg_mempool = mempool_create(t->send_credit_target,
1833 mempool_alloc_slab, mempool_free_slab,
1834 t->sendmsg_cache);
1835 if (!t->sendmsg_mempool)
1836 goto err;
1837
1838 snprintf(name, sizeof(name), "smb_direct_resp_%p", t);
1839 t->recvmsg_cache = kmem_cache_create(name,
1840 sizeof(struct smb_direct_recvmsg) +
1841 t->max_recv_size,
1842 0, SLAB_HWCACHE_ALIGN, NULL);
1843 if (!t->recvmsg_cache)
1844 goto err;
1845
1846 t->recvmsg_mempool =
1847 mempool_create(t->recv_credit_max, mempool_alloc_slab,
1848 mempool_free_slab, t->recvmsg_cache);
1849 if (!t->recvmsg_mempool)
1850 goto err;
1851
1852 INIT_LIST_HEAD(&t->recvmsg_queue);
1853
1854 for (i = 0; i < t->recv_credit_max; i++) {
1855 recvmsg = mempool_alloc(t->recvmsg_mempool, GFP_KERNEL);
1856 if (!recvmsg)
1857 goto err;
1858 recvmsg->transport = t;
1859 list_add(&recvmsg->list, &t->recvmsg_queue);
1860 }
1861 t->count_avail_recvmsg = t->recv_credit_max;
1862
1863 return 0;
1864 err:
1865 smb_direct_destroy_pools(t);
1866 return -ENOMEM;
1867 }
1868
smb_direct_create_qpair(struct smb_direct_transport * t,struct ib_qp_cap * cap)1869 static int smb_direct_create_qpair(struct smb_direct_transport *t,
1870 struct ib_qp_cap *cap)
1871 {
1872 int ret;
1873 struct ib_qp_init_attr qp_attr;
1874 int pages_per_rw;
1875
1876 t->pd = ib_alloc_pd(t->cm_id->device, 0);
1877 if (IS_ERR(t->pd)) {
1878 pr_err("Can't create RDMA PD\n");
1879 ret = PTR_ERR(t->pd);
1880 t->pd = NULL;
1881 return ret;
1882 }
1883
1884 t->send_cq = ib_alloc_cq(t->cm_id->device, t,
1885 smb_direct_send_credit_target + cap->max_rdma_ctxs,
1886 0, IB_POLL_WORKQUEUE);
1887 if (IS_ERR(t->send_cq)) {
1888 pr_err("Can't create RDMA send CQ\n");
1889 ret = PTR_ERR(t->send_cq);
1890 t->send_cq = NULL;
1891 goto err;
1892 }
1893
1894 t->recv_cq = ib_alloc_cq(t->cm_id->device, t,
1895 t->recv_credit_max, 0, IB_POLL_WORKQUEUE);
1896 if (IS_ERR(t->recv_cq)) {
1897 pr_err("Can't create RDMA recv CQ\n");
1898 ret = PTR_ERR(t->recv_cq);
1899 t->recv_cq = NULL;
1900 goto err;
1901 }
1902
1903 memset(&qp_attr, 0, sizeof(qp_attr));
1904 qp_attr.event_handler = smb_direct_qpair_handler;
1905 qp_attr.qp_context = t;
1906 qp_attr.cap = *cap;
1907 qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
1908 qp_attr.qp_type = IB_QPT_RC;
1909 qp_attr.send_cq = t->send_cq;
1910 qp_attr.recv_cq = t->recv_cq;
1911 qp_attr.port_num = ~0;
1912
1913 ret = rdma_create_qp(t->cm_id, t->pd, &qp_attr);
1914 if (ret) {
1915 pr_err("Can't create RDMA QP: %d\n", ret);
1916 goto err;
1917 }
1918
1919 t->qp = t->cm_id->qp;
1920 t->cm_id->event_handler = smb_direct_cm_handler;
1921
1922 pages_per_rw = DIV_ROUND_UP(t->max_rdma_rw_size, PAGE_SIZE) + 1;
1923 if (pages_per_rw > t->cm_id->device->attrs.max_sgl_rd) {
1924 ret = ib_mr_pool_init(t->qp, &t->qp->rdma_mrs,
1925 t->max_rw_credits, IB_MR_TYPE_MEM_REG,
1926 t->pages_per_rw_credit, 0);
1927 if (ret) {
1928 pr_err("failed to init mr pool count %d pages %d\n",
1929 t->max_rw_credits, t->pages_per_rw_credit);
1930 goto err;
1931 }
1932 }
1933
1934 return 0;
1935 err:
1936 if (t->qp) {
1937 ib_destroy_qp(t->qp);
1938 t->qp = NULL;
1939 }
1940 if (t->recv_cq) {
1941 ib_destroy_cq(t->recv_cq);
1942 t->recv_cq = NULL;
1943 }
1944 if (t->send_cq) {
1945 ib_destroy_cq(t->send_cq);
1946 t->send_cq = NULL;
1947 }
1948 if (t->pd) {
1949 ib_dealloc_pd(t->pd);
1950 t->pd = NULL;
1951 }
1952 return ret;
1953 }
1954
smb_direct_prepare(struct ksmbd_transport * t)1955 static int smb_direct_prepare(struct ksmbd_transport *t)
1956 {
1957 struct smb_direct_transport *st = smb_trans_direct_transfort(t);
1958 struct smb_direct_recvmsg *recvmsg;
1959 struct smb_direct_negotiate_req *req;
1960 int ret;
1961
1962 ksmbd_debug(RDMA, "Waiting for SMB_DIRECT negotiate request\n");
1963 ret = wait_event_interruptible_timeout(st->wait_status,
1964 st->negotiation_requested ||
1965 st->status == SMB_DIRECT_CS_DISCONNECTED,
1966 SMB_DIRECT_NEGOTIATE_TIMEOUT * HZ);
1967 if (ret <= 0 || st->status == SMB_DIRECT_CS_DISCONNECTED)
1968 return ret < 0 ? ret : -ETIMEDOUT;
1969
1970 recvmsg = get_first_reassembly(st);
1971 if (!recvmsg)
1972 return -ECONNABORTED;
1973
1974 ret = smb_direct_check_recvmsg(recvmsg);
1975 if (ret == -ECONNABORTED)
1976 goto out;
1977
1978 req = (struct smb_direct_negotiate_req *)recvmsg->packet;
1979 st->max_recv_size = min_t(int, st->max_recv_size,
1980 le32_to_cpu(req->preferred_send_size));
1981 st->max_send_size = min_t(int, st->max_send_size,
1982 le32_to_cpu(req->max_receive_size));
1983 st->max_fragmented_send_size =
1984 le32_to_cpu(req->max_fragmented_size);
1985 st->max_fragmented_recv_size =
1986 (st->recv_credit_max * st->max_recv_size) / 2;
1987
1988 ret = smb_direct_send_negotiate_response(st, ret);
1989 out:
1990 spin_lock_irq(&st->reassembly_queue_lock);
1991 st->reassembly_queue_length--;
1992 list_del(&recvmsg->list);
1993 spin_unlock_irq(&st->reassembly_queue_lock);
1994 put_recvmsg(st, recvmsg);
1995
1996 return ret;
1997 }
1998
smb_direct_connect(struct smb_direct_transport * st)1999 static int smb_direct_connect(struct smb_direct_transport *st)
2000 {
2001 int ret;
2002 struct ib_qp_cap qp_cap;
2003
2004 ret = smb_direct_init_params(st, &qp_cap);
2005 if (ret) {
2006 pr_err("Can't configure RDMA parameters\n");
2007 return ret;
2008 }
2009
2010 ret = smb_direct_create_pools(st);
2011 if (ret) {
2012 pr_err("Can't init RDMA pool: %d\n", ret);
2013 return ret;
2014 }
2015
2016 ret = smb_direct_create_qpair(st, &qp_cap);
2017 if (ret) {
2018 pr_err("Can't accept RDMA client: %d\n", ret);
2019 return ret;
2020 }
2021
2022 ret = smb_direct_prepare_negotiation(st);
2023 if (ret) {
2024 pr_err("Can't negotiate: %d\n", ret);
2025 return ret;
2026 }
2027 return 0;
2028 }
2029
rdma_frwr_is_supported(struct ib_device_attr * attrs)2030 static bool rdma_frwr_is_supported(struct ib_device_attr *attrs)
2031 {
2032 if (!(attrs->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS))
2033 return false;
2034 if (attrs->max_fast_reg_page_list_len == 0)
2035 return false;
2036 return true;
2037 }
2038
smb_direct_handle_connect_request(struct rdma_cm_id * new_cm_id)2039 static int smb_direct_handle_connect_request(struct rdma_cm_id *new_cm_id)
2040 {
2041 struct smb_direct_transport *t;
2042 struct task_struct *handler;
2043 int ret;
2044
2045 if (!rdma_frwr_is_supported(&new_cm_id->device->attrs)) {
2046 ksmbd_debug(RDMA,
2047 "Fast Registration Work Requests is not supported. device capabilities=%llx\n",
2048 new_cm_id->device->attrs.device_cap_flags);
2049 return -EPROTONOSUPPORT;
2050 }
2051
2052 t = alloc_transport(new_cm_id);
2053 if (!t)
2054 return -ENOMEM;
2055
2056 ret = smb_direct_connect(t);
2057 if (ret)
2058 goto out_err;
2059
2060 handler = kthread_run(ksmbd_conn_handler_loop,
2061 KSMBD_TRANS(t)->conn, "ksmbd:r%u",
2062 smb_direct_port);
2063 if (IS_ERR(handler)) {
2064 ret = PTR_ERR(handler);
2065 pr_err("Can't start thread\n");
2066 goto out_err;
2067 }
2068
2069 return 0;
2070 out_err:
2071 free_transport(t);
2072 return ret;
2073 }
2074
smb_direct_listen_handler(struct rdma_cm_id * cm_id,struct rdma_cm_event * event)2075 static int smb_direct_listen_handler(struct rdma_cm_id *cm_id,
2076 struct rdma_cm_event *event)
2077 {
2078 switch (event->event) {
2079 case RDMA_CM_EVENT_CONNECT_REQUEST: {
2080 int ret = smb_direct_handle_connect_request(cm_id);
2081
2082 if (ret) {
2083 pr_err("Can't create transport: %d\n", ret);
2084 return ret;
2085 }
2086
2087 ksmbd_debug(RDMA, "Received connection request. cm_id=%p\n",
2088 cm_id);
2089 break;
2090 }
2091 default:
2092 pr_err("Unexpected listen event. cm_id=%p, event=%s (%d)\n",
2093 cm_id, rdma_event_msg(event->event), event->event);
2094 break;
2095 }
2096 return 0;
2097 }
2098
smb_direct_listen(int port)2099 static int smb_direct_listen(int port)
2100 {
2101 int ret;
2102 struct rdma_cm_id *cm_id;
2103 struct sockaddr_in sin = {
2104 .sin_family = AF_INET,
2105 .sin_addr.s_addr = htonl(INADDR_ANY),
2106 .sin_port = htons(port),
2107 };
2108
2109 cm_id = rdma_create_id(&init_net, smb_direct_listen_handler,
2110 &smb_direct_listener, RDMA_PS_TCP, IB_QPT_RC);
2111 if (IS_ERR(cm_id)) {
2112 pr_err("Can't create cm id: %ld\n", PTR_ERR(cm_id));
2113 return PTR_ERR(cm_id);
2114 }
2115
2116 ret = rdma_bind_addr(cm_id, (struct sockaddr *)&sin);
2117 if (ret) {
2118 pr_err("Can't bind: %d\n", ret);
2119 goto err;
2120 }
2121
2122 smb_direct_listener.cm_id = cm_id;
2123
2124 ret = rdma_listen(cm_id, 10);
2125 if (ret) {
2126 pr_err("Can't listen: %d\n", ret);
2127 goto err;
2128 }
2129 return 0;
2130 err:
2131 smb_direct_listener.cm_id = NULL;
2132 rdma_destroy_id(cm_id);
2133 return ret;
2134 }
2135
smb_direct_ib_client_add(struct ib_device * ib_dev)2136 static int smb_direct_ib_client_add(struct ib_device *ib_dev)
2137 {
2138 struct smb_direct_device *smb_dev;
2139
2140 /* Set 5445 port if device type is iWARP(No IB) */
2141 if (ib_dev->node_type != RDMA_NODE_IB_CA)
2142 smb_direct_port = SMB_DIRECT_PORT_IWARP;
2143
2144 if (!rdma_frwr_is_supported(&ib_dev->attrs))
2145 return 0;
2146
2147 smb_dev = kzalloc(sizeof(*smb_dev), GFP_KERNEL);
2148 if (!smb_dev)
2149 return -ENOMEM;
2150 smb_dev->ib_dev = ib_dev;
2151
2152 write_lock(&smb_direct_device_lock);
2153 list_add(&smb_dev->list, &smb_direct_device_list);
2154 write_unlock(&smb_direct_device_lock);
2155
2156 ksmbd_debug(RDMA, "ib device added: name %s\n", ib_dev->name);
2157 return 0;
2158 }
2159
smb_direct_ib_client_remove(struct ib_device * ib_dev,void * client_data)2160 static void smb_direct_ib_client_remove(struct ib_device *ib_dev,
2161 void *client_data)
2162 {
2163 struct smb_direct_device *smb_dev, *tmp;
2164
2165 write_lock(&smb_direct_device_lock);
2166 list_for_each_entry_safe(smb_dev, tmp, &smb_direct_device_list, list) {
2167 if (smb_dev->ib_dev == ib_dev) {
2168 list_del(&smb_dev->list);
2169 kfree(smb_dev);
2170 break;
2171 }
2172 }
2173 write_unlock(&smb_direct_device_lock);
2174 }
2175
2176 static struct ib_client smb_direct_ib_client = {
2177 .name = "ksmbd_smb_direct_ib",
2178 .add = smb_direct_ib_client_add,
2179 .remove = smb_direct_ib_client_remove,
2180 };
2181
ksmbd_rdma_init(void)2182 int ksmbd_rdma_init(void)
2183 {
2184 int ret;
2185
2186 smb_direct_listener.cm_id = NULL;
2187
2188 ret = ib_register_client(&smb_direct_ib_client);
2189 if (ret) {
2190 pr_err("failed to ib_register_client\n");
2191 return ret;
2192 }
2193
2194 /* When a client is running out of send credits, the credits are
2195 * granted by the server's sending a packet using this queue.
2196 * This avoids the situation that a clients cannot send packets
2197 * for lack of credits
2198 */
2199 smb_direct_wq = alloc_workqueue("ksmbd-smb_direct-wq",
2200 WQ_HIGHPRI | WQ_MEM_RECLAIM, 0);
2201 if (!smb_direct_wq)
2202 return -ENOMEM;
2203
2204 ret = smb_direct_listen(smb_direct_port);
2205 if (ret) {
2206 destroy_workqueue(smb_direct_wq);
2207 smb_direct_wq = NULL;
2208 pr_err("Can't listen: %d\n", ret);
2209 return ret;
2210 }
2211
2212 ksmbd_debug(RDMA, "init RDMA listener. cm_id=%p\n",
2213 smb_direct_listener.cm_id);
2214 return 0;
2215 }
2216
ksmbd_rdma_destroy(void)2217 void ksmbd_rdma_destroy(void)
2218 {
2219 if (!smb_direct_listener.cm_id)
2220 return;
2221
2222 ib_unregister_client(&smb_direct_ib_client);
2223 rdma_destroy_id(smb_direct_listener.cm_id);
2224
2225 smb_direct_listener.cm_id = NULL;
2226
2227 if (smb_direct_wq) {
2228 destroy_workqueue(smb_direct_wq);
2229 smb_direct_wq = NULL;
2230 }
2231 }
2232
ksmbd_rdma_capable_netdev(struct net_device * netdev)2233 bool ksmbd_rdma_capable_netdev(struct net_device *netdev)
2234 {
2235 struct smb_direct_device *smb_dev;
2236 int i;
2237 bool rdma_capable = false;
2238
2239 read_lock(&smb_direct_device_lock);
2240 list_for_each_entry(smb_dev, &smb_direct_device_list, list) {
2241 for (i = 0; i < smb_dev->ib_dev->phys_port_cnt; i++) {
2242 struct net_device *ndev;
2243
2244 if (smb_dev->ib_dev->ops.get_netdev) {
2245 ndev = smb_dev->ib_dev->ops.get_netdev(
2246 smb_dev->ib_dev, i + 1);
2247 if (!ndev)
2248 continue;
2249
2250 if (ndev == netdev) {
2251 dev_put(ndev);
2252 rdma_capable = true;
2253 goto out;
2254 }
2255 dev_put(ndev);
2256 /* if ib_dev does not implement ops.get_netdev
2257 * check for matching infiniband GUID in hw_addr
2258 */
2259 } else if (netdev->type == ARPHRD_INFINIBAND) {
2260 struct netdev_hw_addr *ha;
2261 union ib_gid gid;
2262 u32 port_num;
2263 int ret;
2264
2265 netdev_hw_addr_list_for_each(
2266 ha, &netdev->dev_addrs) {
2267 memcpy(&gid, ha->addr + 4, sizeof(gid));
2268 ret = ib_find_gid(smb_dev->ib_dev, &gid,
2269 &port_num, NULL);
2270 if (!ret) {
2271 rdma_capable = true;
2272 goto out;
2273 }
2274 }
2275 }
2276 }
2277 }
2278 out:
2279 read_unlock(&smb_direct_device_lock);
2280
2281 if (rdma_capable == false) {
2282 struct ib_device *ibdev;
2283
2284 ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_UNKNOWN);
2285 if (ibdev) {
2286 if (rdma_frwr_is_supported(&ibdev->attrs))
2287 rdma_capable = true;
2288 ib_device_put(ibdev);
2289 }
2290 }
2291
2292 return rdma_capable;
2293 }
2294
2295 static const struct ksmbd_transport_ops ksmbd_smb_direct_transport_ops = {
2296 .prepare = smb_direct_prepare,
2297 .disconnect = smb_direct_disconnect,
2298 .shutdown = smb_direct_shutdown,
2299 .writev = smb_direct_writev,
2300 .read = smb_direct_read,
2301 .rdma_read = smb_direct_rdma_read,
2302 .rdma_write = smb_direct_rdma_write,
2303 };
2304