1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3 * Copyright (c) 2025 Stefan Metzmacher
4 */
5
6 #ifndef __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_SOCKET_H__
7 #define __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_SOCKET_H__
8
9 #include <rdma/rw.h>
10
11 enum smbdirect_socket_status {
12 SMBDIRECT_SOCKET_CREATED,
13 SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED,
14 SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING,
15 SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED,
16 SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED,
17 SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING,
18 SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED,
19 SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED,
20 SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING,
21 SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED,
22 SMBDIRECT_SOCKET_NEGOTIATE_NEEDED,
23 SMBDIRECT_SOCKET_NEGOTIATE_RUNNING,
24 SMBDIRECT_SOCKET_NEGOTIATE_FAILED,
25 SMBDIRECT_SOCKET_CONNECTED,
26 SMBDIRECT_SOCKET_ERROR,
27 SMBDIRECT_SOCKET_DISCONNECTING,
28 SMBDIRECT_SOCKET_DISCONNECTED,
29 SMBDIRECT_SOCKET_DESTROYED
30 };
31
32 static __always_inline
smbdirect_socket_status_string(enum smbdirect_socket_status status)33 const char *smbdirect_socket_status_string(enum smbdirect_socket_status status)
34 {
35 switch (status) {
36 case SMBDIRECT_SOCKET_CREATED:
37 return "CREATED";
38 case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED:
39 return "RESOLVE_ADDR_NEEDED";
40 case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING:
41 return "RESOLVE_ADDR_RUNNING";
42 case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED:
43 return "RESOLVE_ADDR_FAILED";
44 case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED:
45 return "RESOLVE_ROUTE_NEEDED";
46 case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING:
47 return "RESOLVE_ROUTE_RUNNING";
48 case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED:
49 return "RESOLVE_ROUTE_FAILED";
50 case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED:
51 return "RDMA_CONNECT_NEEDED";
52 case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING:
53 return "RDMA_CONNECT_RUNNING";
54 case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED:
55 return "RDMA_CONNECT_FAILED";
56 case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED:
57 return "NEGOTIATE_NEEDED";
58 case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING:
59 return "NEGOTIATE_RUNNING";
60 case SMBDIRECT_SOCKET_NEGOTIATE_FAILED:
61 return "NEGOTIATE_FAILED";
62 case SMBDIRECT_SOCKET_CONNECTED:
63 return "CONNECTED";
64 case SMBDIRECT_SOCKET_ERROR:
65 return "ERROR";
66 case SMBDIRECT_SOCKET_DISCONNECTING:
67 return "DISCONNECTING";
68 case SMBDIRECT_SOCKET_DISCONNECTED:
69 return "DISCONNECTED";
70 case SMBDIRECT_SOCKET_DESTROYED:
71 return "DESTROYED";
72 }
73
74 return "<unknown>";
75 }
76
77 enum smbdirect_keepalive_status {
78 SMBDIRECT_KEEPALIVE_NONE,
79 SMBDIRECT_KEEPALIVE_PENDING,
80 SMBDIRECT_KEEPALIVE_SENT
81 };
82
83 struct smbdirect_socket {
84 enum smbdirect_socket_status status;
85 wait_queue_head_t status_wait;
86 int first_error;
87
88 /*
89 * This points to the workqueue to
90 * be used for this socket.
91 * It can be per socket (on the client)
92 * or point to a global workqueue (on the server)
93 */
94 struct workqueue_struct *workqueue;
95
96 struct work_struct disconnect_work;
97
98 /* RDMA related */
99 struct {
100 struct rdma_cm_id *cm_id;
101 /*
102 * This is for iWarp MPA v1
103 */
104 bool legacy_iwarp;
105 } rdma;
106
107 /* IB verbs related */
108 struct {
109 struct ib_pd *pd;
110 struct ib_cq *send_cq;
111 struct ib_cq *recv_cq;
112
113 /*
114 * shortcuts for rdma.cm_id->{qp,device};
115 */
116 struct ib_qp *qp;
117 struct ib_device *dev;
118 } ib;
119
120 struct smbdirect_socket_parameters parameters;
121
122 /*
123 * The state for keepalive and timeout handling
124 */
125 struct {
126 enum smbdirect_keepalive_status keepalive;
127 struct work_struct immediate_work;
128 struct delayed_work timer_work;
129 } idle;
130
131 /*
132 * The state for posted send buffers
133 */
134 struct {
135 /*
136 * Memory pools for preallocating
137 * smbdirect_send_io buffers
138 */
139 struct {
140 struct kmem_cache *cache;
141 mempool_t *pool;
142 } mem;
143
144 /*
145 * The credit state for the send side
146 */
147 struct {
148 atomic_t count;
149 wait_queue_head_t wait_queue;
150 } credits;
151
152 /*
153 * The state about posted/pending sends
154 */
155 struct {
156 atomic_t count;
157 /*
158 * woken when count is decremented
159 */
160 wait_queue_head_t dec_wait_queue;
161 /*
162 * woken when count reached zero
163 */
164 wait_queue_head_t zero_wait_queue;
165 } pending;
166 } send_io;
167
168 /*
169 * The state for posted receive buffers
170 */
171 struct {
172 /*
173 * The type of PDU we are expecting
174 */
175 enum {
176 SMBDIRECT_EXPECT_NEGOTIATE_REQ = 1,
177 SMBDIRECT_EXPECT_NEGOTIATE_REP = 2,
178 SMBDIRECT_EXPECT_DATA_TRANSFER = 3,
179 } expected;
180
181 /*
182 * Memory pools for preallocating
183 * smbdirect_recv_io buffers
184 */
185 struct {
186 struct kmem_cache *cache;
187 mempool_t *pool;
188 } mem;
189
190 /*
191 * The list of free smbdirect_recv_io
192 * structures
193 */
194 struct {
195 struct list_head list;
196 spinlock_t lock;
197 } free;
198
199 /*
200 * The state for posted recv_io messages
201 * and the refill work struct.
202 */
203 struct {
204 atomic_t count;
205 struct work_struct refill_work;
206 } posted;
207
208 /*
209 * The credit state for the recv side
210 */
211 struct {
212 u16 target;
213 atomic_t count;
214 } credits;
215
216 /*
217 * The list of arrived non-empty smbdirect_recv_io
218 * structures
219 *
220 * This represents the reassembly queue.
221 */
222 struct {
223 struct list_head list;
224 spinlock_t lock;
225 wait_queue_head_t wait_queue;
226 /* total data length of reassembly queue */
227 int data_length;
228 int queue_length;
229 /* the offset to first buffer in reassembly queue */
230 int first_entry_offset;
231 /*
232 * Indicate if we have received a full packet on the
233 * connection This is used to identify the first SMBD
234 * packet of a assembled payload (SMB packet) in
235 * reassembly queue so we can return a RFC1002 length to
236 * upper layer to indicate the length of the SMB packet
237 * received
238 */
239 bool full_packet_received;
240 } reassembly;
241 } recv_io;
242
243 /*
244 * The state for Memory registrations on the client
245 */
246 struct {
247 enum ib_mr_type type;
248
249 /*
250 * The list of free smbdirect_mr_io
251 * structures
252 */
253 struct {
254 struct list_head list;
255 spinlock_t lock;
256 } all;
257
258 /*
259 * The number of available MRs ready for memory registration
260 */
261 struct {
262 atomic_t count;
263 wait_queue_head_t wait_queue;
264 } ready;
265
266 /*
267 * The number of used MRs
268 */
269 struct {
270 atomic_t count;
271 } used;
272
273 struct work_struct recovery_work;
274
275 /* Used by transport to wait until all MRs are returned */
276 struct {
277 wait_queue_head_t wait_queue;
278 } cleanup;
279 } mr_io;
280
281 /*
282 * The state for RDMA read/write requests on the server
283 */
284 struct {
285 /*
286 * The credit state for the send side
287 */
288 struct {
289 /*
290 * The maximum number of rw credits
291 */
292 size_t max;
293 /*
294 * The number of pages per credit
295 */
296 size_t num_pages;
297 atomic_t count;
298 wait_queue_head_t wait_queue;
299 } credits;
300 } rw_io;
301
302 /*
303 * For debug purposes
304 */
305 struct {
306 u64 get_receive_buffer;
307 u64 put_receive_buffer;
308 u64 enqueue_reassembly_queue;
309 u64 dequeue_reassembly_queue;
310 u64 send_empty;
311 } statistics;
312 };
313
__smbdirect_socket_disabled_work(struct work_struct * work)314 static void __smbdirect_socket_disabled_work(struct work_struct *work)
315 {
316 /*
317 * Should never be called as disable_[delayed_]work_sync() was used.
318 */
319 WARN_ON_ONCE(1);
320 }
321
smbdirect_socket_init(struct smbdirect_socket * sc)322 static __always_inline void smbdirect_socket_init(struct smbdirect_socket *sc)
323 {
324 /*
325 * This also sets status = SMBDIRECT_SOCKET_CREATED
326 */
327 BUILD_BUG_ON(SMBDIRECT_SOCKET_CREATED != 0);
328 memset(sc, 0, sizeof(*sc));
329
330 init_waitqueue_head(&sc->status_wait);
331
332 INIT_WORK(&sc->disconnect_work, __smbdirect_socket_disabled_work);
333 disable_work_sync(&sc->disconnect_work);
334
335 INIT_WORK(&sc->idle.immediate_work, __smbdirect_socket_disabled_work);
336 disable_work_sync(&sc->idle.immediate_work);
337 INIT_DELAYED_WORK(&sc->idle.timer_work, __smbdirect_socket_disabled_work);
338 disable_delayed_work_sync(&sc->idle.timer_work);
339
340 atomic_set(&sc->send_io.credits.count, 0);
341 init_waitqueue_head(&sc->send_io.credits.wait_queue);
342
343 atomic_set(&sc->send_io.pending.count, 0);
344 init_waitqueue_head(&sc->send_io.pending.dec_wait_queue);
345 init_waitqueue_head(&sc->send_io.pending.zero_wait_queue);
346
347 INIT_LIST_HEAD(&sc->recv_io.free.list);
348 spin_lock_init(&sc->recv_io.free.lock);
349
350 atomic_set(&sc->recv_io.posted.count, 0);
351 INIT_WORK(&sc->recv_io.posted.refill_work, __smbdirect_socket_disabled_work);
352 disable_work_sync(&sc->recv_io.posted.refill_work);
353
354 atomic_set(&sc->recv_io.credits.count, 0);
355
356 INIT_LIST_HEAD(&sc->recv_io.reassembly.list);
357 spin_lock_init(&sc->recv_io.reassembly.lock);
358 init_waitqueue_head(&sc->recv_io.reassembly.wait_queue);
359
360 atomic_set(&sc->rw_io.credits.count, 0);
361 init_waitqueue_head(&sc->rw_io.credits.wait_queue);
362
363 spin_lock_init(&sc->mr_io.all.lock);
364 INIT_LIST_HEAD(&sc->mr_io.all.list);
365 atomic_set(&sc->mr_io.ready.count, 0);
366 init_waitqueue_head(&sc->mr_io.ready.wait_queue);
367 atomic_set(&sc->mr_io.used.count, 0);
368 INIT_WORK(&sc->mr_io.recovery_work, __smbdirect_socket_disabled_work);
369 disable_work_sync(&sc->mr_io.recovery_work);
370 init_waitqueue_head(&sc->mr_io.cleanup.wait_queue);
371 }
372
373 struct smbdirect_send_io {
374 struct smbdirect_socket *socket;
375 struct ib_cqe cqe;
376
377 /*
378 * The SGE entries for this work request
379 *
380 * The first points to the packet header
381 */
382 #define SMBDIRECT_SEND_IO_MAX_SGE 6
383 size_t num_sge;
384 struct ib_sge sge[SMBDIRECT_SEND_IO_MAX_SGE];
385
386 /*
387 * Link to the list of sibling smbdirect_send_io
388 * messages.
389 */
390 struct list_head sibling_list;
391 struct ib_send_wr wr;
392
393 /* SMBD packet header follows this structure */
394 u8 packet[];
395 };
396
397 struct smbdirect_send_batch {
398 /*
399 * List of smbdirect_send_io messages
400 */
401 struct list_head msg_list;
402 /*
403 * Number of list entries
404 */
405 size_t wr_cnt;
406
407 /*
408 * Possible remote key invalidation state
409 */
410 bool need_invalidate_rkey;
411 u32 remote_key;
412 };
413
414 struct smbdirect_recv_io {
415 struct smbdirect_socket *socket;
416 struct ib_cqe cqe;
417
418 /*
419 * For now we only use a single SGE
420 * as we have just one large buffer
421 * per posted recv.
422 */
423 #define SMBDIRECT_RECV_IO_MAX_SGE 1
424 struct ib_sge sge;
425
426 /* Link to free or reassembly list */
427 struct list_head list;
428
429 /* Indicate if this is the 1st packet of a payload */
430 bool first_segment;
431
432 /* SMBD packet header and payload follows this structure */
433 u8 packet[];
434 };
435
436 enum smbdirect_mr_state {
437 SMBDIRECT_MR_READY,
438 SMBDIRECT_MR_REGISTERED,
439 SMBDIRECT_MR_INVALIDATED,
440 SMBDIRECT_MR_ERROR
441 };
442
443 struct smbdirect_mr_io {
444 struct smbdirect_socket *socket;
445 struct ib_cqe cqe;
446
447 struct list_head list;
448
449 enum smbdirect_mr_state state;
450 struct ib_mr *mr;
451 struct sg_table sgt;
452 enum dma_data_direction dir;
453 union {
454 struct ib_reg_wr wr;
455 struct ib_send_wr inv_wr;
456 };
457
458 bool need_invalidate;
459 struct completion invalidate_done;
460 };
461
462 struct smbdirect_rw_io {
463 struct smbdirect_socket *socket;
464 struct ib_cqe cqe;
465
466 struct list_head list;
467
468 int error;
469 struct completion *completion;
470
471 struct rdma_rw_ctx rdma_ctx;
472 struct sg_table sgt;
473 struct scatterlist sg_list[];
474 };
475
476 #endif /* __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_SOCKET_H__ */
477