1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * Copyright (c) 2025 Stefan Metzmacher 4 */ 5 6 #ifndef __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_SOCKET_H__ 7 #define __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_SOCKET_H__ 8 9 #include <rdma/rw.h> 10 11 enum smbdirect_socket_status { 12 SMBDIRECT_SOCKET_CREATED, 13 SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED, 14 SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING, 15 SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED, 16 SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED, 17 SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING, 18 SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED, 19 SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED, 20 SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING, 21 SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED, 22 SMBDIRECT_SOCKET_NEGOTIATE_NEEDED, 23 SMBDIRECT_SOCKET_NEGOTIATE_RUNNING, 24 SMBDIRECT_SOCKET_NEGOTIATE_FAILED, 25 SMBDIRECT_SOCKET_CONNECTED, 26 SMBDIRECT_SOCKET_ERROR, 27 SMBDIRECT_SOCKET_DISCONNECTING, 28 SMBDIRECT_SOCKET_DISCONNECTED, 29 SMBDIRECT_SOCKET_DESTROYED 30 }; 31 32 static __always_inline 33 const char *smbdirect_socket_status_string(enum smbdirect_socket_status status) 34 { 35 switch (status) { 36 case SMBDIRECT_SOCKET_CREATED: 37 return "CREATED"; 38 case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED: 39 return "RESOLVE_ADDR_NEEDED"; 40 case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING: 41 return "RESOLVE_ADDR_RUNNING"; 42 case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED: 43 return "RESOLVE_ADDR_FAILED"; 44 case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED: 45 return "RESOLVE_ROUTE_NEEDED"; 46 case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING: 47 return "RESOLVE_ROUTE_RUNNING"; 48 case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED: 49 return "RESOLVE_ROUTE_FAILED"; 50 case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED: 51 return "RDMA_CONNECT_NEEDED"; 52 case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING: 53 return "RDMA_CONNECT_RUNNING"; 54 case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED: 55 return "RDMA_CONNECT_FAILED"; 56 case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED: 57 return "NEGOTIATE_NEEDED"; 58 case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING: 59 return "NEGOTIATE_RUNNING"; 60 case SMBDIRECT_SOCKET_NEGOTIATE_FAILED: 61 return "NEGOTIATE_FAILED"; 62 case SMBDIRECT_SOCKET_CONNECTED: 63 return "CONNECTED"; 64 case SMBDIRECT_SOCKET_ERROR: 65 return "ERROR"; 66 case SMBDIRECT_SOCKET_DISCONNECTING: 67 return "DISCONNECTING"; 68 case SMBDIRECT_SOCKET_DISCONNECTED: 69 return "DISCONNECTED"; 70 case SMBDIRECT_SOCKET_DESTROYED: 71 return "DESTROYED"; 72 } 73 74 return "<unknown>"; 75 } 76 77 enum smbdirect_keepalive_status { 78 SMBDIRECT_KEEPALIVE_NONE, 79 SMBDIRECT_KEEPALIVE_PENDING, 80 SMBDIRECT_KEEPALIVE_SENT 81 }; 82 83 struct smbdirect_socket { 84 enum smbdirect_socket_status status; 85 wait_queue_head_t status_wait; 86 int first_error; 87 88 /* 89 * This points to the workqueue to 90 * be used for this socket. 91 * It can be per socket (on the client) 92 * or point to a global workqueue (on the server) 93 */ 94 struct workqueue_struct *workqueue; 95 96 struct work_struct disconnect_work; 97 98 /* RDMA related */ 99 struct { 100 struct rdma_cm_id *cm_id; 101 /* 102 * This is for iWarp MPA v1 103 */ 104 bool legacy_iwarp; 105 } rdma; 106 107 /* IB verbs related */ 108 struct { 109 struct ib_pd *pd; 110 struct ib_cq *send_cq; 111 struct ib_cq *recv_cq; 112 113 /* 114 * shortcuts for rdma.cm_id->{qp,device}; 115 */ 116 struct ib_qp *qp; 117 struct ib_device *dev; 118 } ib; 119 120 struct smbdirect_socket_parameters parameters; 121 122 /* 123 * The state for keepalive and timeout handling 124 */ 125 struct { 126 enum smbdirect_keepalive_status keepalive; 127 struct work_struct immediate_work; 128 struct delayed_work timer_work; 129 } idle; 130 131 /* 132 * The state for posted send buffers 133 */ 134 struct { 135 /* 136 * Memory pools for preallocating 137 * smbdirect_send_io buffers 138 */ 139 struct { 140 struct kmem_cache *cache; 141 mempool_t *pool; 142 } mem; 143 144 /* 145 * The credit state for the send side 146 */ 147 struct { 148 atomic_t count; 149 wait_queue_head_t wait_queue; 150 } credits; 151 152 /* 153 * The state about posted/pending sends 154 */ 155 struct { 156 atomic_t count; 157 /* 158 * woken when count is decremented 159 */ 160 wait_queue_head_t dec_wait_queue; 161 /* 162 * woken when count reached zero 163 */ 164 wait_queue_head_t zero_wait_queue; 165 } pending; 166 } send_io; 167 168 /* 169 * The state for posted receive buffers 170 */ 171 struct { 172 /* 173 * The type of PDU we are expecting 174 */ 175 enum { 176 SMBDIRECT_EXPECT_NEGOTIATE_REQ = 1, 177 SMBDIRECT_EXPECT_NEGOTIATE_REP = 2, 178 SMBDIRECT_EXPECT_DATA_TRANSFER = 3, 179 } expected; 180 181 /* 182 * Memory pools for preallocating 183 * smbdirect_recv_io buffers 184 */ 185 struct { 186 struct kmem_cache *cache; 187 mempool_t *pool; 188 } mem; 189 190 /* 191 * The list of free smbdirect_recv_io 192 * structures 193 */ 194 struct { 195 struct list_head list; 196 spinlock_t lock; 197 } free; 198 199 /* 200 * The state for posted recv_io messages 201 * and the refill work struct. 202 */ 203 struct { 204 atomic_t count; 205 struct work_struct refill_work; 206 } posted; 207 208 /* 209 * The credit state for the recv side 210 */ 211 struct { 212 u16 target; 213 atomic_t count; 214 } credits; 215 216 /* 217 * The list of arrived non-empty smbdirect_recv_io 218 * structures 219 * 220 * This represents the reassembly queue. 221 */ 222 struct { 223 struct list_head list; 224 spinlock_t lock; 225 wait_queue_head_t wait_queue; 226 /* total data length of reassembly queue */ 227 int data_length; 228 int queue_length; 229 /* the offset to first buffer in reassembly queue */ 230 int first_entry_offset; 231 /* 232 * Indicate if we have received a full packet on the 233 * connection This is used to identify the first SMBD 234 * packet of a assembled payload (SMB packet) in 235 * reassembly queue so we can return a RFC1002 length to 236 * upper layer to indicate the length of the SMB packet 237 * received 238 */ 239 bool full_packet_received; 240 } reassembly; 241 } recv_io; 242 243 /* 244 * The state for Memory registrations on the client 245 */ 246 struct { 247 enum ib_mr_type type; 248 249 /* 250 * The list of free smbdirect_mr_io 251 * structures 252 */ 253 struct { 254 struct list_head list; 255 spinlock_t lock; 256 } all; 257 258 /* 259 * The number of available MRs ready for memory registration 260 */ 261 struct { 262 atomic_t count; 263 wait_queue_head_t wait_queue; 264 } ready; 265 266 /* 267 * The number of used MRs 268 */ 269 struct { 270 atomic_t count; 271 } used; 272 273 struct work_struct recovery_work; 274 275 /* Used by transport to wait until all MRs are returned */ 276 struct { 277 wait_queue_head_t wait_queue; 278 } cleanup; 279 } mr_io; 280 281 /* 282 * The state for RDMA read/write requests on the server 283 */ 284 struct { 285 /* 286 * The credit state for the send side 287 */ 288 struct { 289 /* 290 * The maximum number of rw credits 291 */ 292 size_t max; 293 /* 294 * The number of pages per credit 295 */ 296 size_t num_pages; 297 atomic_t count; 298 wait_queue_head_t wait_queue; 299 } credits; 300 } rw_io; 301 302 /* 303 * For debug purposes 304 */ 305 struct { 306 u64 get_receive_buffer; 307 u64 put_receive_buffer; 308 u64 enqueue_reassembly_queue; 309 u64 dequeue_reassembly_queue; 310 u64 send_empty; 311 } statistics; 312 }; 313 314 static void __smbdirect_socket_disabled_work(struct work_struct *work) 315 { 316 /* 317 * Should never be called as disable_[delayed_]work_sync() was used. 318 */ 319 WARN_ON_ONCE(1); 320 } 321 322 static __always_inline void smbdirect_socket_init(struct smbdirect_socket *sc) 323 { 324 /* 325 * This also sets status = SMBDIRECT_SOCKET_CREATED 326 */ 327 BUILD_BUG_ON(SMBDIRECT_SOCKET_CREATED != 0); 328 memset(sc, 0, sizeof(*sc)); 329 330 init_waitqueue_head(&sc->status_wait); 331 332 INIT_WORK(&sc->disconnect_work, __smbdirect_socket_disabled_work); 333 disable_work_sync(&sc->disconnect_work); 334 335 INIT_WORK(&sc->idle.immediate_work, __smbdirect_socket_disabled_work); 336 disable_work_sync(&sc->idle.immediate_work); 337 INIT_DELAYED_WORK(&sc->idle.timer_work, __smbdirect_socket_disabled_work); 338 disable_delayed_work_sync(&sc->idle.timer_work); 339 340 atomic_set(&sc->send_io.credits.count, 0); 341 init_waitqueue_head(&sc->send_io.credits.wait_queue); 342 343 atomic_set(&sc->send_io.pending.count, 0); 344 init_waitqueue_head(&sc->send_io.pending.dec_wait_queue); 345 init_waitqueue_head(&sc->send_io.pending.zero_wait_queue); 346 347 INIT_LIST_HEAD(&sc->recv_io.free.list); 348 spin_lock_init(&sc->recv_io.free.lock); 349 350 atomic_set(&sc->recv_io.posted.count, 0); 351 INIT_WORK(&sc->recv_io.posted.refill_work, __smbdirect_socket_disabled_work); 352 disable_work_sync(&sc->recv_io.posted.refill_work); 353 354 atomic_set(&sc->recv_io.credits.count, 0); 355 356 INIT_LIST_HEAD(&sc->recv_io.reassembly.list); 357 spin_lock_init(&sc->recv_io.reassembly.lock); 358 init_waitqueue_head(&sc->recv_io.reassembly.wait_queue); 359 360 atomic_set(&sc->rw_io.credits.count, 0); 361 init_waitqueue_head(&sc->rw_io.credits.wait_queue); 362 363 spin_lock_init(&sc->mr_io.all.lock); 364 INIT_LIST_HEAD(&sc->mr_io.all.list); 365 atomic_set(&sc->mr_io.ready.count, 0); 366 init_waitqueue_head(&sc->mr_io.ready.wait_queue); 367 atomic_set(&sc->mr_io.used.count, 0); 368 INIT_WORK(&sc->mr_io.recovery_work, __smbdirect_socket_disabled_work); 369 disable_work_sync(&sc->mr_io.recovery_work); 370 init_waitqueue_head(&sc->mr_io.cleanup.wait_queue); 371 } 372 373 struct smbdirect_send_io { 374 struct smbdirect_socket *socket; 375 struct ib_cqe cqe; 376 377 /* 378 * The SGE entries for this work request 379 * 380 * The first points to the packet header 381 */ 382 #define SMBDIRECT_SEND_IO_MAX_SGE 6 383 size_t num_sge; 384 struct ib_sge sge[SMBDIRECT_SEND_IO_MAX_SGE]; 385 386 /* 387 * Link to the list of sibling smbdirect_send_io 388 * messages. 389 */ 390 struct list_head sibling_list; 391 struct ib_send_wr wr; 392 393 /* SMBD packet header follows this structure */ 394 u8 packet[]; 395 }; 396 397 struct smbdirect_send_batch { 398 /* 399 * List of smbdirect_send_io messages 400 */ 401 struct list_head msg_list; 402 /* 403 * Number of list entries 404 */ 405 size_t wr_cnt; 406 407 /* 408 * Possible remote key invalidation state 409 */ 410 bool need_invalidate_rkey; 411 u32 remote_key; 412 }; 413 414 struct smbdirect_recv_io { 415 struct smbdirect_socket *socket; 416 struct ib_cqe cqe; 417 418 /* 419 * For now we only use a single SGE 420 * as we have just one large buffer 421 * per posted recv. 422 */ 423 #define SMBDIRECT_RECV_IO_MAX_SGE 1 424 struct ib_sge sge; 425 426 /* Link to free or reassembly list */ 427 struct list_head list; 428 429 /* Indicate if this is the 1st packet of a payload */ 430 bool first_segment; 431 432 /* SMBD packet header and payload follows this structure */ 433 u8 packet[]; 434 }; 435 436 enum smbdirect_mr_state { 437 SMBDIRECT_MR_READY, 438 SMBDIRECT_MR_REGISTERED, 439 SMBDIRECT_MR_INVALIDATED, 440 SMBDIRECT_MR_ERROR 441 }; 442 443 struct smbdirect_mr_io { 444 struct smbdirect_socket *socket; 445 struct ib_cqe cqe; 446 447 struct list_head list; 448 449 enum smbdirect_mr_state state; 450 struct ib_mr *mr; 451 struct sg_table sgt; 452 enum dma_data_direction dir; 453 union { 454 struct ib_reg_wr wr; 455 struct ib_send_wr inv_wr; 456 }; 457 458 bool need_invalidate; 459 struct completion invalidate_done; 460 }; 461 462 struct smbdirect_rw_io { 463 struct smbdirect_socket *socket; 464 struct ib_cqe cqe; 465 466 struct list_head list; 467 468 int error; 469 struct completion *completion; 470 471 struct rdma_rw_ctx rdma_ctx; 472 struct sg_table sgt; 473 struct scatterlist sg_list[]; 474 }; 475 476 #endif /* __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_SOCKET_H__ */ 477