Lines Matching full:sc
18 struct smbdirect_socket *sc = &conn->socket; in smbd_get_parameters() local
20 return &sc->parameters; in smbd_get_parameters()
24 struct smbdirect_socket *sc);
26 struct smbdirect_socket *sc,
28 static int allocate_receive_buffers(struct smbdirect_socket *sc, int num_buf);
29 static void destroy_receive_buffers(struct smbdirect_socket *sc);
32 struct smbdirect_socket *sc,
35 struct smbdirect_socket *sc);
38 struct smbdirect_socket *sc,
41 static int smbd_post_send_empty(struct smbdirect_socket *sc);
43 static void destroy_mr_list(struct smbdirect_socket *sc);
44 static int allocate_mr_list(struct smbdirect_socket *sc);
168 static void smbd_disconnect_wake_up_all(struct smbdirect_socket *sc) in smbd_disconnect_wake_up_all() argument
174 wake_up_all(&sc->status_wait); in smbd_disconnect_wake_up_all()
175 wake_up_all(&sc->send_io.credits.wait_queue); in smbd_disconnect_wake_up_all()
176 wake_up_all(&sc->send_io.pending.dec_wait_queue); in smbd_disconnect_wake_up_all()
177 wake_up_all(&sc->send_io.pending.zero_wait_queue); in smbd_disconnect_wake_up_all()
178 wake_up_all(&sc->recv_io.reassembly.wait_queue); in smbd_disconnect_wake_up_all()
179 wake_up_all(&sc->mr_io.ready.wait_queue); in smbd_disconnect_wake_up_all()
180 wake_up_all(&sc->mr_io.cleanup.wait_queue); in smbd_disconnect_wake_up_all()
185 struct smbdirect_socket *sc = in smbd_disconnect_rdma_work() local
193 disable_work(&sc->disconnect_work); in smbd_disconnect_rdma_work()
194 disable_work(&sc->recv_io.posted.refill_work); in smbd_disconnect_rdma_work()
195 disable_work(&sc->mr_io.recovery_work); in smbd_disconnect_rdma_work()
196 disable_work(&sc->idle.immediate_work); in smbd_disconnect_rdma_work()
197 disable_delayed_work(&sc->idle.timer_work); in smbd_disconnect_rdma_work()
199 if (sc->first_error == 0) in smbd_disconnect_rdma_work()
200 sc->first_error = -ECONNABORTED; in smbd_disconnect_rdma_work()
202 switch (sc->status) { in smbd_disconnect_rdma_work()
208 sc->status = SMBDIRECT_SOCKET_DISCONNECTING; in smbd_disconnect_rdma_work()
209 rdma_disconnect(sc->rdma.cm_id); in smbd_disconnect_rdma_work()
226 sc->status = SMBDIRECT_SOCKET_DISCONNECTED; in smbd_disconnect_rdma_work()
239 smbd_disconnect_wake_up_all(sc); in smbd_disconnect_rdma_work()
242 static void smbd_disconnect_rdma_connection(struct smbdirect_socket *sc) in smbd_disconnect_rdma_connection() argument
249 disable_work(&sc->recv_io.posted.refill_work); in smbd_disconnect_rdma_connection()
250 disable_work(&sc->mr_io.recovery_work); in smbd_disconnect_rdma_connection()
251 disable_work(&sc->idle.immediate_work); in smbd_disconnect_rdma_connection()
252 disable_delayed_work(&sc->idle.timer_work); in smbd_disconnect_rdma_connection()
254 if (sc->first_error == 0) in smbd_disconnect_rdma_connection()
255 sc->first_error = -ECONNABORTED; in smbd_disconnect_rdma_connection()
257 switch (sc->status) { in smbd_disconnect_rdma_connection()
273 sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED; in smbd_disconnect_rdma_connection()
278 sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED; in smbd_disconnect_rdma_connection()
283 sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED; in smbd_disconnect_rdma_connection()
288 sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED; in smbd_disconnect_rdma_connection()
293 sc->status = SMBDIRECT_SOCKET_ERROR; in smbd_disconnect_rdma_connection()
301 smbd_disconnect_wake_up_all(sc); in smbd_disconnect_rdma_connection()
303 queue_work(sc->workqueue, &sc->disconnect_work); in smbd_disconnect_rdma_connection()
310 struct smbdirect_socket *sc = id->context; in smbd_conn_upcall() local
311 struct smbdirect_socket_parameters *sp = &sc->parameters; in smbd_conn_upcall()
321 WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING); in smbd_conn_upcall()
322 sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED; in smbd_conn_upcall()
323 wake_up(&sc->status_wait); in smbd_conn_upcall()
327 WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING); in smbd_conn_upcall()
328 sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED; in smbd_conn_upcall()
329 wake_up(&sc->status_wait); in smbd_conn_upcall()
334 WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING); in smbd_conn_upcall()
335 sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED; in smbd_conn_upcall()
336 smbd_disconnect_rdma_work(&sc->disconnect_work); in smbd_conn_upcall()
341 WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING); in smbd_conn_upcall()
342 sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED; in smbd_conn_upcall()
343 smbd_disconnect_rdma_work(&sc->disconnect_work); in smbd_conn_upcall()
407 sc->rdma.legacy_iwarp = true; in smbd_conn_upcall()
427 WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING); in smbd_conn_upcall()
428 sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED; in smbd_conn_upcall()
429 wake_up(&sc->status_wait); in smbd_conn_upcall()
436 WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING); in smbd_conn_upcall()
437 sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED; in smbd_conn_upcall()
438 smbd_disconnect_rdma_work(&sc->disconnect_work); in smbd_conn_upcall()
444 if (sc->status == SMBDIRECT_SOCKET_NEGOTIATE_FAILED) { in smbd_conn_upcall()
448 sc->status = SMBDIRECT_SOCKET_DISCONNECTED; in smbd_conn_upcall()
449 smbd_disconnect_rdma_work(&sc->disconnect_work); in smbd_conn_upcall()
465 struct smbdirect_socket *sc = context; in smbd_qp_async_error_upcall() local
468 ib_event_msg(event->event), event->device->name, sc); in smbd_qp_async_error_upcall()
473 smbd_disconnect_rdma_connection(sc); in smbd_qp_async_error_upcall()
497 struct smbdirect_socket *sc = request->socket; in send_done() local
503 ib_dma_unmap_single(sc->ib.dev, in send_done()
512 mempool_free(request, sc->send_io.mem.pool); in send_done()
513 smbd_disconnect_rdma_connection(sc); in send_done()
517 if (atomic_dec_and_test(&sc->send_io.pending.count)) in send_done()
518 wake_up(&sc->send_io.pending.zero_wait_queue); in send_done()
520 wake_up(&sc->send_io.pending.dec_wait_queue); in send_done()
522 mempool_free(request, sc->send_io.mem.pool); in send_done()
543 struct smbdirect_socket *sc = response->socket; in process_negotiation_response() local
544 struct smbdirect_socket_parameters *sp = &sc->parameters; in process_negotiation_response()
563 sc->recv_io.credits.target = le16_to_cpu(packet->credits_requested); in process_negotiation_response()
564 sc->recv_io.credits.target = min_t(u16, sc->recv_io.credits.target, sp->recv_credit_max); in process_negotiation_response()
570 atomic_set(&sc->send_io.credits.count, le16_to_cpu(packet->credits_granted)); in process_negotiation_response()
602 sc->recv_io.expected = SMBDIRECT_EXPECT_DATA_TRANSFER; in process_negotiation_response()
610 struct smbdirect_socket *sc = in smbd_post_send_credits() local
613 if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { in smbd_post_send_credits()
617 if (sc->recv_io.credits.target > in smbd_post_send_credits()
618 atomic_read(&sc->recv_io.credits.count)) { in smbd_post_send_credits()
620 response = get_receive_buffer(sc); in smbd_post_send_credits()
625 rc = smbd_post_recv(sc, response); in smbd_post_send_credits()
629 put_receive_buffer(sc, response); in smbd_post_send_credits()
633 atomic_inc(&sc->recv_io.posted.count); in smbd_post_send_credits()
638 if (atomic_read(&sc->recv_io.credits.count) < in smbd_post_send_credits()
639 sc->recv_io.credits.target - 1) { in smbd_post_send_credits()
641 queue_work(sc->workqueue, &sc->idle.immediate_work); in smbd_post_send_credits()
651 struct smbdirect_socket *sc = response->socket; in recv_done() local
652 struct smbdirect_socket_parameters *sp = &sc->parameters; in recv_done()
661 response, sc->recv_io.expected, in recv_done()
682 sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE; in recv_done()
683 mod_delayed_work(sc->workqueue, &sc->idle.timer_work, in recv_done()
686 switch (sc->recv_io.expected) { in recv_done()
690 sc->recv_io.reassembly.full_packet_received = true; in recv_done()
693 put_receive_buffer(sc, response); in recv_done()
694 WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_NEGOTIATE_RUNNING); in recv_done()
696 sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED; in recv_done()
697 smbd_disconnect_rdma_connection(sc); in recv_done()
699 sc->status = SMBDIRECT_SOCKET_CONNECTED; in recv_done()
700 wake_up(&sc->status_wait); in recv_done()
726 if (sc->recv_io.reassembly.full_packet_received) in recv_done()
730 sc->recv_io.reassembly.full_packet_received = false; in recv_done()
732 sc->recv_io.reassembly.full_packet_received = true; in recv_done()
735 atomic_dec(&sc->recv_io.posted.count); in recv_done()
736 atomic_dec(&sc->recv_io.credits.count); in recv_done()
737 old_recv_credit_target = sc->recv_io.credits.target; in recv_done()
738 sc->recv_io.credits.target = in recv_done()
740 sc->recv_io.credits.target = in recv_done()
741 min_t(u16, sc->recv_io.credits.target, sp->recv_credit_max); in recv_done()
742 sc->recv_io.credits.target = in recv_done()
743 max_t(u16, sc->recv_io.credits.target, 1); in recv_done()
746 &sc->send_io.credits.count); in recv_done()
751 wake_up(&sc->send_io.credits.wait_queue); in recv_done()
764 queue_work(sc->workqueue, &sc->idle.immediate_work); in recv_done()
772 if (sc->recv_io.credits.target > old_recv_credit_target) in recv_done()
773 queue_work(sc->workqueue, &sc->recv_io.posted.refill_work); in recv_done()
775 enqueue_reassembly(sc, response, data_length); in recv_done()
776 wake_up(&sc->recv_io.reassembly.wait_queue); in recv_done()
778 put_receive_buffer(sc, response); in recv_done()
790 log_rdma_recv(ERR, "unexpected response type=%d\n", sc->recv_io.expected); in recv_done()
791 WARN_ON_ONCE(sc->recv_io.expected != SMBDIRECT_EXPECT_DATA_TRANSFER); in recv_done()
793 put_receive_buffer(sc, response); in recv_done()
794 smbd_disconnect_rdma_connection(sc); in recv_done()
798 struct smbdirect_socket *sc, in smbd_create_id() argument
801 struct smbdirect_socket_parameters *sp = &sc->parameters; in smbd_create_id()
806 id = rdma_create_id(&init_net, smbd_conn_upcall, sc, in smbd_create_id()
821 WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED); in smbd_create_id()
822 sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING; in smbd_create_id()
830 sc->status_wait, in smbd_create_id()
831 sc->status != SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING, in smbd_create_id()
838 if (sc->status == SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING) { in smbd_create_id()
843 if (sc->status != SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED) { in smbd_create_id()
849 WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED); in smbd_create_id()
850 sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING; in smbd_create_id()
857 sc->status_wait, in smbd_create_id()
858 sc->status != SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING, in smbd_create_id()
865 if (sc->status == SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING) { in smbd_create_id()
870 if (sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED) { in smbd_create_id()
898 struct smbdirect_socket *sc, in smbd_ia_open() argument
901 struct smbdirect_socket_parameters *sp = &sc->parameters; in smbd_ia_open()
904 WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED); in smbd_ia_open()
905 sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED; in smbd_ia_open()
907 sc->rdma.cm_id = smbd_create_id(sc, dstaddr, port); in smbd_ia_open()
908 if (IS_ERR(sc->rdma.cm_id)) { in smbd_ia_open()
909 rc = PTR_ERR(sc->rdma.cm_id); in smbd_ia_open()
912 sc->ib.dev = sc->rdma.cm_id->device; in smbd_ia_open()
914 if (!frwr_is_supported(&sc->ib.dev->attrs)) { in smbd_ia_open()
917 sc->ib.dev->attrs.device_cap_flags, in smbd_ia_open()
918 sc->ib.dev->attrs.max_fast_reg_page_list_len); in smbd_ia_open()
924 sc->ib.dev->attrs.max_fast_reg_page_list_len); in smbd_ia_open()
925 sc->mr_io.type = IB_MR_TYPE_MEM_REG; in smbd_ia_open()
926 if (sc->ib.dev->attrs.kernel_cap_flags & IBK_SG_GAPS_REG) in smbd_ia_open()
927 sc->mr_io.type = IB_MR_TYPE_SG_GAPS; in smbd_ia_open()
932 rdma_destroy_id(sc->rdma.cm_id); in smbd_ia_open()
933 sc->rdma.cm_id = NULL; in smbd_ia_open()
945 static int smbd_post_send_negotiate_req(struct smbdirect_socket *sc) in smbd_post_send_negotiate_req() argument
947 struct smbdirect_socket_parameters *sp = &sc->parameters; in smbd_post_send_negotiate_req()
953 request = mempool_alloc(sc->send_io.mem.pool, GFP_KERNEL); in smbd_post_send_negotiate_req()
957 request->socket = sc; in smbd_post_send_negotiate_req()
971 sc->ib.dev, (void *)packet, in smbd_post_send_negotiate_req()
973 if (ib_dma_mapping_error(sc->ib.dev, request->sge[0].addr)) { in smbd_post_send_negotiate_req()
979 request->sge[0].lkey = sc->ib.pd->local_dma_lkey; in smbd_post_send_negotiate_req()
982 sc->ib.dev, request->sge[0].addr, in smbd_post_send_negotiate_req()
998 atomic_inc(&sc->send_io.pending.count); in smbd_post_send_negotiate_req()
999 rc = ib_post_send(sc->ib.qp, &send_wr, NULL); in smbd_post_send_negotiate_req()
1005 atomic_dec(&sc->send_io.pending.count); in smbd_post_send_negotiate_req()
1006 ib_dma_unmap_single(sc->ib.dev, request->sge[0].addr, in smbd_post_send_negotiate_req()
1009 smbd_disconnect_rdma_connection(sc); in smbd_post_send_negotiate_req()
1012 mempool_free(request, sc->send_io.mem.pool); in smbd_post_send_negotiate_req()
1024 static int manage_credits_prior_sending(struct smbdirect_socket *sc) in manage_credits_prior_sending() argument
1028 if (atomic_read(&sc->recv_io.credits.count) >= sc->recv_io.credits.target) in manage_credits_prior_sending()
1031 new_credits = atomic_read(&sc->recv_io.posted.count); in manage_credits_prior_sending()
1035 new_credits -= atomic_read(&sc->recv_io.credits.count); in manage_credits_prior_sending()
1051 static int manage_keep_alive_before_sending(struct smbdirect_socket *sc) in manage_keep_alive_before_sending() argument
1053 struct smbdirect_socket_parameters *sp = &sc->parameters; in manage_keep_alive_before_sending()
1055 if (sc->idle.keepalive == SMBDIRECT_KEEPALIVE_PENDING) { in manage_keep_alive_before_sending()
1056 sc->idle.keepalive = SMBDIRECT_KEEPALIVE_SENT; in manage_keep_alive_before_sending()
1061 mod_delayed_work(sc->workqueue, &sc->idle.timer_work, in manage_keep_alive_before_sending()
1069 static int smbd_post_send(struct smbdirect_socket *sc, in smbd_post_send() argument
1080 sc->ib.dev, in smbd_post_send()
1095 rc = ib_post_send(sc->ib.qp, &send_wr, NULL); in smbd_post_send()
1098 smbd_disconnect_rdma_connection(sc); in smbd_post_send()
1105 static int smbd_post_send_iter(struct smbdirect_socket *sc, in smbd_post_send_iter() argument
1109 struct smbdirect_socket_parameters *sp = &sc->parameters; in smbd_post_send_iter()
1119 rc = wait_event_interruptible(sc->send_io.credits.wait_queue, in smbd_post_send_iter()
1120 atomic_read(&sc->send_io.credits.count) > 0 || in smbd_post_send_iter()
1121 sc->status != SMBDIRECT_SOCKET_CONNECTED); in smbd_post_send_iter()
1125 if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { in smbd_post_send_iter()
1130 if (unlikely(atomic_dec_return(&sc->send_io.credits.count) < 0)) { in smbd_post_send_iter()
1131 atomic_inc(&sc->send_io.credits.count); in smbd_post_send_iter()
1136 wait_event(sc->send_io.pending.dec_wait_queue, in smbd_post_send_iter()
1137 atomic_read(&sc->send_io.pending.count) < sp->send_credit_target || in smbd_post_send_iter()
1138 sc->status != SMBDIRECT_SOCKET_CONNECTED); in smbd_post_send_iter()
1140 if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { in smbd_post_send_iter()
1146 if (unlikely(atomic_inc_return(&sc->send_io.pending.count) > in smbd_post_send_iter()
1148 atomic_dec(&sc->send_io.pending.count); in smbd_post_send_iter()
1152 request = mempool_alloc(sc->send_io.mem.pool, GFP_KERNEL); in smbd_post_send_iter()
1158 request->socket = sc; in smbd_post_send_iter()
1168 request->sge[0].addr = ib_dma_map_single(sc->ib.dev, in smbd_post_send_iter()
1172 if (ib_dma_mapping_error(sc->ib.dev, request->sge[0].addr)) { in smbd_post_send_iter()
1178 request->sge[0].lkey = sc->ib.pd->local_dma_lkey; in smbd_post_send_iter()
1187 .device = sc->ib.dev, in smbd_post_send_iter()
1188 .local_dma_lkey = sc->ib.pd->local_dma_lkey, in smbd_post_send_iter()
1208 new_credits = manage_credits_prior_sending(sc); in smbd_post_send_iter()
1209 atomic_add(new_credits, &sc->recv_io.credits.count); in smbd_post_send_iter()
1213 if (manage_keep_alive_before_sending(sc)) in smbd_post_send_iter()
1232 rc = smbd_post_send(sc, request); in smbd_post_send_iter()
1239 ib_dma_unmap_single(sc->ib.dev, in smbd_post_send_iter()
1243 mempool_free(request, sc->send_io.mem.pool); in smbd_post_send_iter()
1246 atomic_sub(new_credits, &sc->recv_io.credits.count); in smbd_post_send_iter()
1249 if (atomic_dec_and_test(&sc->send_io.pending.count)) in smbd_post_send_iter()
1250 wake_up(&sc->send_io.pending.zero_wait_queue); in smbd_post_send_iter()
1254 atomic_inc(&sc->send_io.credits.count); in smbd_post_send_iter()
1265 static int smbd_post_send_empty(struct smbdirect_socket *sc) in smbd_post_send_empty() argument
1269 sc->statistics.send_empty++; in smbd_post_send_empty()
1270 return smbd_post_send_iter(sc, NULL, &remaining_data_length); in smbd_post_send_empty()
1273 static int smbd_post_send_full_iter(struct smbdirect_socket *sc, in smbd_post_send_full_iter() argument
1286 rc = smbd_post_send_iter(sc, iter, _remaining_data_length); in smbd_post_send_full_iter()
1300 struct smbdirect_socket *sc, struct smbdirect_recv_io *response) in smbd_post_recv() argument
1302 struct smbdirect_socket_parameters *sp = &sc->parameters; in smbd_post_recv()
1307 sc->ib.dev, response->packet, in smbd_post_recv()
1309 if (ib_dma_mapping_error(sc->ib.dev, response->sge.addr)) in smbd_post_recv()
1313 response->sge.lkey = sc->ib.pd->local_dma_lkey; in smbd_post_recv()
1322 rc = ib_post_recv(sc->ib.qp, &recv_wr, NULL); in smbd_post_recv()
1324 ib_dma_unmap_single(sc->ib.dev, response->sge.addr, in smbd_post_recv()
1327 smbd_disconnect_rdma_connection(sc); in smbd_post_recv()
1335 static int smbd_negotiate(struct smbdirect_socket *sc) in smbd_negotiate() argument
1337 struct smbdirect_socket_parameters *sp = &sc->parameters; in smbd_negotiate()
1339 struct smbdirect_recv_io *response = get_receive_buffer(sc); in smbd_negotiate()
1341 WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_NEGOTIATE_NEEDED); in smbd_negotiate()
1342 sc->status = SMBDIRECT_SOCKET_NEGOTIATE_RUNNING; in smbd_negotiate()
1344 sc->recv_io.expected = SMBDIRECT_EXPECT_NEGOTIATE_REP; in smbd_negotiate()
1345 rc = smbd_post_recv(sc, response); in smbd_negotiate()
1350 put_receive_buffer(sc, response); in smbd_negotiate()
1354 rc = smbd_post_send_negotiate_req(sc); in smbd_negotiate()
1359 sc->status_wait, in smbd_negotiate()
1360 sc->status != SMBDIRECT_SOCKET_NEGOTIATE_RUNNING, in smbd_negotiate()
1364 if (sc->status == SMBDIRECT_SOCKET_CONNECTED) in smbd_negotiate()
1388 struct smbdirect_socket *sc, in enqueue_reassembly() argument
1394 spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); in enqueue_reassembly()
1395 list_add_tail(&response->list, &sc->recv_io.reassembly.list); in enqueue_reassembly()
1396 sc->recv_io.reassembly.queue_length++; in enqueue_reassembly()
1404 sc->recv_io.reassembly.data_length += data_length; in enqueue_reassembly()
1405 spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); in enqueue_reassembly()
1406 sc->statistics.enqueue_reassembly_queue++; in enqueue_reassembly()
1414 static struct smbdirect_recv_io *_get_first_reassembly(struct smbdirect_socket *sc) in _get_first_reassembly() argument
1418 if (!list_empty(&sc->recv_io.reassembly.list)) { in _get_first_reassembly()
1420 &sc->recv_io.reassembly.list, in _get_first_reassembly()
1432 static struct smbdirect_recv_io *get_receive_buffer(struct smbdirect_socket *sc) in get_receive_buffer() argument
1437 spin_lock_irqsave(&sc->recv_io.free.lock, flags); in get_receive_buffer()
1438 if (!list_empty(&sc->recv_io.free.list)) { in get_receive_buffer()
1440 &sc->recv_io.free.list, in get_receive_buffer()
1443 sc->statistics.get_receive_buffer++; in get_receive_buffer()
1445 spin_unlock_irqrestore(&sc->recv_io.free.lock, flags); in get_receive_buffer()
1457 struct smbdirect_socket *sc, struct smbdirect_recv_io *response) in put_receive_buffer() argument
1462 ib_dma_unmap_single(sc->ib.dev, in put_receive_buffer()
1469 spin_lock_irqsave(&sc->recv_io.free.lock, flags); in put_receive_buffer()
1470 list_add_tail(&response->list, &sc->recv_io.free.list); in put_receive_buffer()
1471 sc->statistics.put_receive_buffer++; in put_receive_buffer()
1472 spin_unlock_irqrestore(&sc->recv_io.free.lock, flags); in put_receive_buffer()
1474 queue_work(sc->workqueue, &sc->recv_io.posted.refill_work); in put_receive_buffer()
1478 static int allocate_receive_buffers(struct smbdirect_socket *sc, int num_buf) in allocate_receive_buffers() argument
1484 response = mempool_alloc(sc->recv_io.mem.pool, GFP_KERNEL); in allocate_receive_buffers()
1488 response->socket = sc; in allocate_receive_buffers()
1490 list_add_tail(&response->list, &sc->recv_io.free.list); in allocate_receive_buffers()
1496 while (!list_empty(&sc->recv_io.free.list)) { in allocate_receive_buffers()
1498 &sc->recv_io.free.list, in allocate_receive_buffers()
1502 mempool_free(response, sc->recv_io.mem.pool); in allocate_receive_buffers()
1507 static void destroy_receive_buffers(struct smbdirect_socket *sc) in destroy_receive_buffers() argument
1511 while ((response = get_receive_buffer(sc))) in destroy_receive_buffers()
1512 mempool_free(response, sc->recv_io.mem.pool); in destroy_receive_buffers()
1517 struct smbdirect_socket *sc = in send_immediate_empty_message() local
1520 if (sc->status != SMBDIRECT_SOCKET_CONNECTED) in send_immediate_empty_message()
1524 smbd_post_send_empty(sc); in send_immediate_empty_message()
1530 struct smbdirect_socket *sc = in idle_connection_timer() local
1532 struct smbdirect_socket_parameters *sp = &sc->parameters; in idle_connection_timer()
1534 if (sc->idle.keepalive != SMBDIRECT_KEEPALIVE_NONE) { in idle_connection_timer()
1536 "error status sc->idle.keepalive=%d\n", in idle_connection_timer()
1537 sc->idle.keepalive); in idle_connection_timer()
1538 smbd_disconnect_rdma_connection(sc); in idle_connection_timer()
1542 if (sc->status != SMBDIRECT_SOCKET_CONNECTED) in idle_connection_timer()
1549 sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING; in idle_connection_timer()
1550 mod_delayed_work(sc->workqueue, &sc->idle.timer_work, in idle_connection_timer()
1553 queue_work(sc->workqueue, &sc->idle.immediate_work); in idle_connection_timer()
1564 struct smbdirect_socket *sc; in smbd_destroy() local
1572 sc = &info->socket; in smbd_destroy()
1575 disable_work_sync(&sc->disconnect_work); in smbd_destroy()
1578 if (sc->status < SMBDIRECT_SOCKET_DISCONNECTING) in smbd_destroy()
1579 smbd_disconnect_rdma_work(&sc->disconnect_work); in smbd_destroy()
1580 if (sc->status < SMBDIRECT_SOCKET_DISCONNECTED) { in smbd_destroy()
1582 wait_event(sc->status_wait, sc->status == SMBDIRECT_SOCKET_DISCONNECTED); in smbd_destroy()
1593 smbd_disconnect_wake_up_all(sc); in smbd_destroy()
1596 disable_work_sync(&sc->recv_io.posted.refill_work); in smbd_destroy()
1599 ib_drain_qp(sc->ib.qp); in smbd_destroy()
1600 rdma_destroy_qp(sc->rdma.cm_id); in smbd_destroy()
1601 sc->ib.qp = NULL; in smbd_destroy()
1604 disable_delayed_work_sync(&sc->idle.timer_work); in smbd_destroy()
1606 disable_work_sync(&sc->idle.immediate_work); in smbd_destroy()
1611 spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); in smbd_destroy()
1612 response = _get_first_reassembly(sc); in smbd_destroy()
1616 &sc->recv_io.reassembly.lock, flags); in smbd_destroy()
1617 put_receive_buffer(sc, response); in smbd_destroy()
1620 &sc->recv_io.reassembly.lock, flags); in smbd_destroy()
1622 sc->recv_io.reassembly.data_length = 0; in smbd_destroy()
1625 destroy_receive_buffers(sc); in smbd_destroy()
1628 destroy_mr_list(sc); in smbd_destroy()
1630 ib_free_cq(sc->ib.send_cq); in smbd_destroy()
1631 ib_free_cq(sc->ib.recv_cq); in smbd_destroy()
1632 ib_dealloc_pd(sc->ib.pd); in smbd_destroy()
1633 rdma_destroy_id(sc->rdma.cm_id); in smbd_destroy()
1636 mempool_destroy(sc->send_io.mem.pool); in smbd_destroy()
1637 kmem_cache_destroy(sc->send_io.mem.cache); in smbd_destroy()
1639 mempool_destroy(sc->recv_io.mem.pool); in smbd_destroy()
1640 kmem_cache_destroy(sc->recv_io.mem.cache); in smbd_destroy()
1642 sc->status = SMBDIRECT_SOCKET_DESTROYED; in smbd_destroy()
1644 destroy_workqueue(sc->workqueue); in smbd_destroy()
1686 static void destroy_caches(struct smbdirect_socket *sc) in destroy_caches() argument
1688 destroy_receive_buffers(sc); in destroy_caches()
1689 mempool_destroy(sc->recv_io.mem.pool); in destroy_caches()
1690 kmem_cache_destroy(sc->recv_io.mem.cache); in destroy_caches()
1691 mempool_destroy(sc->send_io.mem.pool); in destroy_caches()
1692 kmem_cache_destroy(sc->send_io.mem.cache); in destroy_caches()
1696 static int allocate_caches(struct smbdirect_socket *sc) in allocate_caches() argument
1698 struct smbdirect_socket_parameters *sp = &sc->parameters; in allocate_caches()
1705 scnprintf(name, MAX_NAME_LEN, "smbdirect_send_io_%p", sc); in allocate_caches()
1706 sc->send_io.mem.cache = in allocate_caches()
1712 if (!sc->send_io.mem.cache) in allocate_caches()
1715 sc->send_io.mem.pool = in allocate_caches()
1717 mempool_free_slab, sc->send_io.mem.cache); in allocate_caches()
1718 if (!sc->send_io.mem.pool) in allocate_caches()
1721 scnprintf(name, MAX_NAME_LEN, "smbdirect_recv_io_%p", sc); in allocate_caches()
1729 sc->recv_io.mem.cache = in allocate_caches()
1733 if (!sc->recv_io.mem.cache) in allocate_caches()
1736 sc->recv_io.mem.pool = in allocate_caches()
1738 mempool_free_slab, sc->recv_io.mem.cache); in allocate_caches()
1739 if (!sc->recv_io.mem.pool) in allocate_caches()
1742 rc = allocate_receive_buffers(sc, sp->recv_credit_max); in allocate_caches()
1751 mempool_destroy(sc->recv_io.mem.pool); in allocate_caches()
1753 kmem_cache_destroy(sc->recv_io.mem.cache); in allocate_caches()
1755 mempool_destroy(sc->send_io.mem.pool); in allocate_caches()
1757 kmem_cache_destroy(sc->send_io.mem.cache); in allocate_caches()
1767 struct smbdirect_socket *sc; in _smbd_get_connection() local
1780 sc = &info->socket; in _smbd_get_connection()
1781 scnprintf(wq_name, ARRAY_SIZE(wq_name), "smbd_%p", sc); in _smbd_get_connection()
1785 smbdirect_socket_init(sc); in _smbd_get_connection()
1786 sc->workqueue = workqueue; in _smbd_get_connection()
1787 sp = &sc->parameters; in _smbd_get_connection()
1789 INIT_WORK(&sc->disconnect_work, smbd_disconnect_rdma_work); in _smbd_get_connection()
1806 rc = smbd_ia_open(sc, dstaddr, port); in _smbd_get_connection()
1812 if (sp->send_credit_target > sc->ib.dev->attrs.max_cqe || in _smbd_get_connection()
1813 sp->send_credit_target > sc->ib.dev->attrs.max_qp_wr) { in _smbd_get_connection()
1816 sc->ib.dev->attrs.max_cqe, in _smbd_get_connection()
1817 sc->ib.dev->attrs.max_qp_wr); in _smbd_get_connection()
1821 if (sp->recv_credit_max > sc->ib.dev->attrs.max_cqe || in _smbd_get_connection()
1822 sp->recv_credit_max > sc->ib.dev->attrs.max_qp_wr) { in _smbd_get_connection()
1825 sc->ib.dev->attrs.max_cqe, in _smbd_get_connection()
1826 sc->ib.dev->attrs.max_qp_wr); in _smbd_get_connection()
1830 if (sc->ib.dev->attrs.max_send_sge < SMBDIRECT_SEND_IO_MAX_SGE || in _smbd_get_connection()
1831 sc->ib.dev->attrs.max_recv_sge < SMBDIRECT_RECV_IO_MAX_SGE) { in _smbd_get_connection()
1835 sc->ib.dev->name, in _smbd_get_connection()
1836 sc->ib.dev->attrs.max_send_sge, in _smbd_get_connection()
1837 sc->ib.dev->attrs.max_recv_sge); in _smbd_get_connection()
1841 sc->ib.pd = ib_alloc_pd(sc->ib.dev, 0); in _smbd_get_connection()
1842 if (IS_ERR(sc->ib.pd)) { in _smbd_get_connection()
1843 rc = PTR_ERR(sc->ib.pd); in _smbd_get_connection()
1844 sc->ib.pd = NULL; in _smbd_get_connection()
1849 sc->ib.send_cq = in _smbd_get_connection()
1850 ib_alloc_cq_any(sc->ib.dev, sc, in _smbd_get_connection()
1852 if (IS_ERR(sc->ib.send_cq)) { in _smbd_get_connection()
1853 sc->ib.send_cq = NULL; in _smbd_get_connection()
1857 sc->ib.recv_cq = in _smbd_get_connection()
1858 ib_alloc_cq_any(sc->ib.dev, sc, in _smbd_get_connection()
1860 if (IS_ERR(sc->ib.recv_cq)) { in _smbd_get_connection()
1861 sc->ib.recv_cq = NULL; in _smbd_get_connection()
1867 qp_attr.qp_context = sc; in _smbd_get_connection()
1875 qp_attr.send_cq = sc->ib.send_cq; in _smbd_get_connection()
1876 qp_attr.recv_cq = sc->ib.recv_cq; in _smbd_get_connection()
1879 rc = rdma_create_qp(sc->rdma.cm_id, sc->ib.pd, &qp_attr); in _smbd_get_connection()
1884 sc->ib.qp = sc->rdma.cm_id->qp; in _smbd_get_connection()
1888 sc->ib.dev->attrs.max_qp_rd_atom); in _smbd_get_connection()
1897 sc->ib.dev->ops.get_port_immutable( in _smbd_get_connection()
1898 sc->ib.dev, sc->rdma.cm_id->port_num, &port_immutable); in _smbd_get_connection()
1916 WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED); in _smbd_get_connection()
1917 sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING; in _smbd_get_connection()
1918 rc = rdma_connect(sc->rdma.cm_id, &conn_param); in _smbd_get_connection()
1925 sc->status_wait, in _smbd_get_connection()
1926 sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING, in _smbd_get_connection()
1929 if (sc->status != SMBDIRECT_SOCKET_NEGOTIATE_NEEDED) { in _smbd_get_connection()
1936 rc = allocate_caches(sc); in _smbd_get_connection()
1942 INIT_WORK(&sc->idle.immediate_work, send_immediate_empty_message); in _smbd_get_connection()
1943 INIT_DELAYED_WORK(&sc->idle.timer_work, idle_connection_timer); in _smbd_get_connection()
1948 sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING; in _smbd_get_connection()
1949 mod_delayed_work(sc->workqueue, &sc->idle.timer_work, in _smbd_get_connection()
1952 INIT_WORK(&sc->recv_io.posted.refill_work, smbd_post_send_credits); in _smbd_get_connection()
1954 rc = smbd_negotiate(sc); in _smbd_get_connection()
1960 rc = allocate_mr_list(sc); in _smbd_get_connection()
1975 disable_delayed_work_sync(&sc->idle.timer_work); in _smbd_get_connection()
1976 destroy_caches(sc); in _smbd_get_connection()
1977 sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED; in _smbd_get_connection()
1978 rdma_disconnect(sc->rdma.cm_id); in _smbd_get_connection()
1979 wait_event(sc->status_wait, in _smbd_get_connection()
1980 sc->status == SMBDIRECT_SOCKET_DISCONNECTED); in _smbd_get_connection()
1984 rdma_destroy_qp(sc->rdma.cm_id); in _smbd_get_connection()
1988 if (sc->ib.send_cq) in _smbd_get_connection()
1989 ib_free_cq(sc->ib.send_cq); in _smbd_get_connection()
1990 if (sc->ib.recv_cq) in _smbd_get_connection()
1991 ib_free_cq(sc->ib.recv_cq); in _smbd_get_connection()
1993 ib_dealloc_pd(sc->ib.pd); in _smbd_get_connection()
1997 rdma_destroy_id(sc->rdma.cm_id); in _smbd_get_connection()
2000 destroy_workqueue(sc->workqueue); in _smbd_get_connection()
2050 struct smbdirect_socket *sc = &info->socket; in smbd_recv() local
2067 log_read(INFO, "size=%zd sc->recv_io.reassembly.data_length=%d\n", size, in smbd_recv()
2068 sc->recv_io.reassembly.data_length); in smbd_recv()
2069 if (sc->recv_io.reassembly.data_length >= size) { in smbd_recv()
2082 queue_length = sc->recv_io.reassembly.queue_length; in smbd_recv()
2085 offset = sc->recv_io.reassembly.first_entry_offset; in smbd_recv()
2087 response = _get_first_reassembly(sc); in smbd_recv()
2133 &sc->recv_io.reassembly.lock, flags); in smbd_recv()
2136 &sc->recv_io.reassembly.lock, flags); in smbd_recv()
2139 sc->statistics.dequeue_reassembly_queue++; in smbd_recv()
2140 put_receive_buffer(sc, response); in smbd_recv()
2154 spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); in smbd_recv()
2155 sc->recv_io.reassembly.data_length -= data_read; in smbd_recv()
2156 sc->recv_io.reassembly.queue_length -= queue_removed; in smbd_recv()
2157 spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); in smbd_recv()
2159 sc->recv_io.reassembly.first_entry_offset = offset; in smbd_recv()
2161 data_read, sc->recv_io.reassembly.data_length, in smbd_recv()
2162 sc->recv_io.reassembly.first_entry_offset); in smbd_recv()
2169 sc->recv_io.reassembly.wait_queue, in smbd_recv()
2170 sc->recv_io.reassembly.data_length >= size || in smbd_recv()
2171 sc->status != SMBDIRECT_SOCKET_CONNECTED); in smbd_recv()
2176 if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { in smbd_recv()
2194 struct smbdirect_socket *sc = &info->socket; in smbd_send() local
2195 struct smbdirect_socket_parameters *sp = &sc->parameters; in smbd_send()
2201 if (sc->status != SMBDIRECT_SOCKET_CONNECTED) in smbd_send()
2242 rc = smbd_post_send_full_iter(sc, &iter, &remaining_data_length); in smbd_send()
2248 rc = smbd_post_send_full_iter(sc, &rqst->rq_iter, in smbd_send()
2263 wait_event(sc->send_io.pending.zero_wait_queue, in smbd_send()
2264 atomic_read(&sc->send_io.pending.count) == 0 || in smbd_send()
2265 sc->status != SMBDIRECT_SOCKET_CONNECTED); in smbd_send()
2267 if (sc->status != SMBDIRECT_SOCKET_CONNECTED && rc == 0) in smbd_send()
2277 struct smbdirect_socket *sc = mr->socket; in register_mr_done() local
2281 smbd_disconnect_rdma_connection(sc); in register_mr_done()
2296 struct smbdirect_socket *sc = in smbd_mr_recovery_work() local
2298 struct smbdirect_socket_parameters *sp = &sc->parameters; in smbd_mr_recovery_work()
2302 list_for_each_entry(smbdirect_mr, &sc->mr_io.all.list, list) { in smbd_mr_recovery_work()
2311 smbd_disconnect_rdma_connection(sc); in smbd_mr_recovery_work()
2316 sc->ib.pd, sc->mr_io.type, in smbd_mr_recovery_work()
2320 sc->mr_io.type, in smbd_mr_recovery_work()
2322 smbd_disconnect_rdma_connection(sc); in smbd_mr_recovery_work()
2338 if (atomic_inc_return(&sc->mr_io.ready.count) == 1) in smbd_mr_recovery_work()
2339 wake_up(&sc->mr_io.ready.wait_queue); in smbd_mr_recovery_work()
2345 struct smbdirect_socket *sc = mr->socket; in smbd_mr_disable_locked() local
2355 ib_dma_unmap_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir); in smbd_mr_disable_locked()
2383 static void destroy_mr_list(struct smbdirect_socket *sc) in destroy_mr_list() argument
2389 disable_work_sync(&sc->mr_io.recovery_work); in destroy_mr_list()
2391 spin_lock_irqsave(&sc->mr_io.all.lock, flags); in destroy_mr_list()
2392 list_splice_tail_init(&sc->mr_io.all.list, &all_list); in destroy_mr_list()
2393 spin_unlock_irqrestore(&sc->mr_io.all.lock, flags); in destroy_mr_list()
2431 static int allocate_mr_list(struct smbdirect_socket *sc) in allocate_mr_list() argument
2433 struct smbdirect_socket_parameters *sp = &sc->parameters; in allocate_mr_list()
2454 mr->mr = ib_alloc_mr(sc->ib.pd, in allocate_mr_list()
2455 sc->mr_io.type, in allocate_mr_list()
2460 sc->mr_io.type, sp->max_frmr_depth); in allocate_mr_list()
2473 mr->socket = sc; in allocate_mr_list()
2475 list_add_tail(&mr->list, &sc->mr_io.all.list); in allocate_mr_list()
2476 atomic_inc(&sc->mr_io.ready.count); in allocate_mr_list()
2479 INIT_WORK(&sc->mr_io.recovery_work, smbd_mr_recovery_work); in allocate_mr_list()
2489 destroy_mr_list(sc); in allocate_mr_list()
2501 static struct smbdirect_mr_io *get_mr(struct smbdirect_socket *sc) in get_mr() argument
2507 rc = wait_event_interruptible(sc->mr_io.ready.wait_queue, in get_mr()
2508 atomic_read(&sc->mr_io.ready.count) || in get_mr()
2509 sc->status != SMBDIRECT_SOCKET_CONNECTED); in get_mr()
2515 if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { in get_mr()
2516 log_rdma_mr(ERR, "sc->status=%x\n", sc->status); in get_mr()
2520 spin_lock_irqsave(&sc->mr_io.all.lock, flags); in get_mr()
2521 list_for_each_entry(ret, &sc->mr_io.all.list, list) { in get_mr()
2525 spin_unlock_irqrestore(&sc->mr_io.all.lock, flags); in get_mr()
2526 atomic_dec(&sc->mr_io.ready.count); in get_mr()
2527 atomic_inc(&sc->mr_io.used.count); in get_mr()
2532 spin_unlock_irqrestore(&sc->mr_io.all.lock, flags); in get_mr()
2569 struct smbdirect_socket *sc = &info->socket; in smbd_register_mr() local
2570 struct smbdirect_socket_parameters *sp = &sc->parameters; in smbd_register_mr()
2583 mr = get_mr(sc); in smbd_register_mr()
2600 rc = ib_dma_map_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir); in smbd_register_mr()
2633 rc = ib_post_send(sc->ib.qp, ®_wr->wr, NULL); in smbd_register_mr()
2650 ib_dma_unmap_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir); in smbd_register_mr()
2655 if (atomic_dec_and_test(&sc->mr_io.used.count)) in smbd_register_mr()
2656 wake_up(&sc->mr_io.cleanup.wait_queue); in smbd_register_mr()
2658 smbd_disconnect_rdma_connection(sc); in smbd_register_mr()
2705 struct smbdirect_socket *sc = mr->socket; in smbd_deregister_mr() local
2711 if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { in smbd_deregister_mr()
2729 rc = ib_post_send(sc->ib.qp, wr, NULL); in smbd_deregister_mr()
2733 smbd_disconnect_rdma_connection(sc); in smbd_deregister_mr()
2746 ib_dma_unmap_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir); in smbd_deregister_mr()
2752 if (atomic_inc_return(&sc->mr_io.ready.count) == 1) in smbd_deregister_mr()
2753 wake_up(&sc->mr_io.ready.wait_queue); in smbd_deregister_mr()
2759 queue_work(sc->workqueue, &sc->mr_io.recovery_work); in smbd_deregister_mr()
2762 if (atomic_dec_and_test(&sc->mr_io.used.count)) in smbd_deregister_mr()
2763 wake_up(&sc->mr_io.cleanup.wait_queue); in smbd_deregister_mr()