117 	struct smbdirect_socket *sc;  in get_smbd_max_read_write_size()  local
124 	sc = &t->socket;  in get_smbd_max_read_write_size()
125 	sp = &sc->parameters;  in get_smbd_max_read_write_size()
136 static void smb_direct_destroy_pools(struct smbdirect_socket *sc);
138 static int smb_direct_post_send_data(struct smbdirect_socket *sc,
150 smbdirect_recv_io *get_free_recvmsg(struct smbdirect_socket *sc)  in get_free_recvmsg()  argument
155 	spin_lock_irqsave(&sc->recv_io.free.lock, flags);  in get_free_recvmsg()
156 	if (!list_empty(&sc->recv_io.free.list)) {  in get_free_recvmsg()
157 		recvmsg = list_first_entry(&sc->recv_io.free.list,  in get_free_recvmsg()
162 	spin_unlock_irqrestore(&sc->recv_io.free.lock, flags);  in get_free_recvmsg()
166 static void put_recvmsg(struct smbdirect_socket *sc,  in put_recvmsg()  argument
172 		ib_dma_unmap_single(sc->ib.dev,  in put_recvmsg()
179 	spin_lock_irqsave(&sc->recv_io.free.lock, flags);  in put_recvmsg()
180 	list_add(&recvmsg->list, &sc->recv_io.free.list);  in put_recvmsg()
181 	spin_unlock_irqrestore(&sc->recv_io.free.lock, flags);  in put_recvmsg()
183 	queue_work(sc->workqueue, &sc->recv_io.posted.refill_work);  in put_recvmsg()
186 static void enqueue_reassembly(struct smbdirect_socket *sc,  in enqueue_reassembly()  argument
192 	spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags);  in enqueue_reassembly()
193 	list_add_tail(&recvmsg->list, &sc->recv_io.reassembly.list);  in enqueue_reassembly()
194 	sc->recv_io.reassembly.queue_length++;  in enqueue_reassembly()
202 	sc->recv_io.reassembly.data_length += data_length;  in enqueue_reassembly()
203 	spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags);  in enqueue_reassembly()
206 static struct smbdirect_recv_io *get_first_reassembly(struct smbdirect_socket *sc)  in get_first_reassembly()  argument
208 	if (!list_empty(&sc->recv_io.reassembly.list))  in get_first_reassembly()
209 		return list_first_entry(&sc->recv_io.reassembly.list,  in get_first_reassembly()
215 static void smb_direct_disconnect_wake_up_all(struct smbdirect_socket *sc)  in smb_direct_disconnect_wake_up_all()  argument
221 	wake_up_all(&sc->status_wait);  in smb_direct_disconnect_wake_up_all()
222 	wake_up_all(&sc->send_io.credits.wait_queue);  in smb_direct_disconnect_wake_up_all()
223 	wake_up_all(&sc->send_io.pending.zero_wait_queue);  in smb_direct_disconnect_wake_up_all()
224 	wake_up_all(&sc->recv_io.reassembly.wait_queue);  in smb_direct_disconnect_wake_up_all()
225 	wake_up_all(&sc->rw_io.credits.wait_queue);  in smb_direct_disconnect_wake_up_all()
230 	struct smbdirect_socket *sc =  in smb_direct_disconnect_rdma_work()  local
238 	disable_work(&sc->disconnect_work);  in smb_direct_disconnect_rdma_work()
239 	disable_work(&sc->recv_io.posted.refill_work);  in smb_direct_disconnect_rdma_work()
240 	disable_delayed_work(&sc->idle.timer_work);  in smb_direct_disconnect_rdma_work()
241 	disable_work(&sc->idle.immediate_work);  in smb_direct_disconnect_rdma_work()
243 	if (sc->first_error == 0)  in smb_direct_disconnect_rdma_work()
244 		sc->first_error = -ECONNABORTED;  in smb_direct_disconnect_rdma_work()
246 	switch (sc->status) {  in smb_direct_disconnect_rdma_work()
252 		sc->status = SMBDIRECT_SOCKET_DISCONNECTING;  in smb_direct_disconnect_rdma_work()
253 		rdma_disconnect(sc->rdma.cm_id);  in smb_direct_disconnect_rdma_work()
270 		sc->status = SMBDIRECT_SOCKET_DISCONNECTED;  in smb_direct_disconnect_rdma_work()
283 	smb_direct_disconnect_wake_up_all(sc);  in smb_direct_disconnect_rdma_work()
287 smb_direct_disconnect_rdma_connection(struct smbdirect_socket *sc)  in smb_direct_disconnect_rdma_connection()  argument
294 	disable_work(&sc->recv_io.posted.refill_work);  in smb_direct_disconnect_rdma_connection()
295 	disable_work(&sc->idle.immediate_work);  in smb_direct_disconnect_rdma_connection()
296 	disable_delayed_work(&sc->idle.timer_work);  in smb_direct_disconnect_rdma_connection()
298 	if (sc->first_error == 0)  in smb_direct_disconnect_rdma_connection()
299 		sc->first_error = -ECONNABORTED;  in smb_direct_disconnect_rdma_connection()
301 	switch (sc->status) {  in smb_direct_disconnect_rdma_connection()
317 		sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED;  in smb_direct_disconnect_rdma_connection()
322 		sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED;  in smb_direct_disconnect_rdma_connection()
327 		sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED;  in smb_direct_disconnect_rdma_connection()
332 		sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED;  in smb_direct_disconnect_rdma_connection()
337 		sc->status = SMBDIRECT_SOCKET_ERROR;  in smb_direct_disconnect_rdma_connection()
345 	smb_direct_disconnect_wake_up_all(sc);  in smb_direct_disconnect_rdma_connection()
347 	queue_work(sc->workqueue, &sc->disconnect_work);  in smb_direct_disconnect_rdma_connection()
352 	struct smbdirect_socket *sc =  in smb_direct_send_immediate_work()  local
355 	if (sc->status != SMBDIRECT_SOCKET_CONNECTED)  in smb_direct_send_immediate_work()
358 	smb_direct_post_send_data(sc, NULL, NULL, 0, 0);  in smb_direct_send_immediate_work()
363 	struct smbdirect_socket *sc =  in smb_direct_idle_connection_timer()  local
365 	struct smbdirect_socket_parameters *sp = &sc->parameters;  in smb_direct_idle_connection_timer()
367 	if (sc->idle.keepalive != SMBDIRECT_KEEPALIVE_NONE) {  in smb_direct_idle_connection_timer()
368 		smb_direct_disconnect_rdma_connection(sc);  in smb_direct_idle_connection_timer()
372 	if (sc->status != SMBDIRECT_SOCKET_CONNECTED)  in smb_direct_idle_connection_timer()
379 	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING;  in smb_direct_idle_connection_timer()
380 	mod_delayed_work(sc->workqueue, &sc->idle.timer_work,  in smb_direct_idle_connection_timer()
382 	queue_work(sc->workqueue, &sc->idle.immediate_work);  in smb_direct_idle_connection_timer()
388 	struct smbdirect_socket *sc;  in alloc_transport()  local
395 	sc = &t->socket;  in alloc_transport()
396 	smbdirect_socket_init(sc);  in alloc_transport()
397 	sp = &sc->parameters;  in alloc_transport()
399 	sc->workqueue = smb_direct_wq;  in alloc_transport()
401 	INIT_WORK(&sc->disconnect_work, smb_direct_disconnect_rdma_work);  in alloc_transport()
415 	sc->rdma.cm_id = cm_id;  in alloc_transport()
416 	cm_id->context = sc;  in alloc_transport()
418 	sc->ib.dev = sc->rdma.cm_id->device;  in alloc_transport()
420 	INIT_WORK(&sc->recv_io.posted.refill_work,  in alloc_transport()
422 	INIT_WORK(&sc->idle.immediate_work, smb_direct_send_immediate_work);  in alloc_transport()
423 	INIT_DELAYED_WORK(&sc->idle.timer_work, smb_direct_idle_connection_timer);  in alloc_transport()
449 	struct smbdirect_socket *sc = &t->socket;  in free_transport()  local
452 	disable_work_sync(&sc->disconnect_work);  in free_transport()
453 	if (sc->status < SMBDIRECT_SOCKET_DISCONNECTING) {  in free_transport()
454 		smb_direct_disconnect_rdma_work(&sc->disconnect_work);  in free_transport()
455 		wait_event_interruptible(sc->status_wait,  in free_transport()
456 					 sc->status == SMBDIRECT_SOCKET_DISCONNECTED);  in free_transport()
466 	smb_direct_disconnect_wake_up_all(sc);  in free_transport()
468 	disable_work_sync(&sc->recv_io.posted.refill_work);  in free_transport()
469 	disable_delayed_work_sync(&sc->idle.timer_work);  in free_transport()
470 	disable_work_sync(&sc->idle.immediate_work);  in free_transport()
472 	if (sc->ib.qp) {  in free_transport()
473 		ib_drain_qp(sc->ib.qp);  in free_transport()
474 		ib_mr_pool_destroy(sc->ib.qp, &sc->ib.qp->rdma_mrs);  in free_transport()
475 		sc->ib.qp = NULL;  in free_transport()
476 		rdma_destroy_qp(sc->rdma.cm_id);  in free_transport()
483 		spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags);  in free_transport()
484 		recvmsg = get_first_reassembly(sc);  in free_transport()
487 			spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags);  in free_transport()
488 			put_recvmsg(sc, recvmsg);  in free_transport()
490 			spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags);  in free_transport()
493 	sc->recv_io.reassembly.data_length = 0;  in free_transport()
495 	if (sc->ib.send_cq)  in free_transport()
496 		ib_free_cq(sc->ib.send_cq);  in free_transport()
497 	if (sc->ib.recv_cq)  in free_transport()
498 		ib_free_cq(sc->ib.recv_cq);  in free_transport()
499 	if (sc->ib.pd)  in free_transport()
500 		ib_dealloc_pd(sc->ib.pd);  in free_transport()
501 	if (sc->rdma.cm_id)  in free_transport()
502 		rdma_destroy_id(sc->rdma.cm_id);  in free_transport()
504 	smb_direct_destroy_pools(sc);  in free_transport()
509 *smb_direct_alloc_sendmsg(struct smbdirect_socket *sc)  in smb_direct_alloc_sendmsg()  argument
513 	msg = mempool_alloc(sc->send_io.mem.pool, KSMBD_DEFAULT_GFP);  in smb_direct_alloc_sendmsg()
516 	msg->socket = sc;  in smb_direct_alloc_sendmsg()
522 static void smb_direct_free_sendmsg(struct smbdirect_socket *sc,  in smb_direct_free_sendmsg()  argument
528 		ib_dma_unmap_single(sc->ib.dev,  in smb_direct_free_sendmsg()
532 			ib_dma_unmap_page(sc->ib.dev,  in smb_direct_free_sendmsg()
536 	mempool_free(msg, sc->send_io.mem.pool);  in smb_direct_free_sendmsg()
541 	struct smbdirect_socket *sc = recvmsg->socket;  in smb_direct_check_recvmsg()  local
543 	switch (sc->recv_io.expected) {  in smb_direct_check_recvmsg()
591 	struct smbdirect_socket *sc;  in recv_done()  local
595 	sc = recvmsg->socket;  in recv_done()
596 	sp = &sc->parameters;  in recv_done()
599 		put_recvmsg(sc, recvmsg);  in recv_done()
604 			smb_direct_disconnect_rdma_connection(sc);  in recv_done()
620 	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE;  in recv_done()
621 	mod_delayed_work(sc->workqueue, &sc->idle.timer_work,  in recv_done()
624 	switch (sc->recv_io.expected) {  in recv_done()
627 			put_recvmsg(sc, recvmsg);  in recv_done()
628 			smb_direct_disconnect_rdma_connection(sc);  in recv_done()
631 		sc->recv_io.reassembly.full_packet_received = true;  in recv_done()
632 		WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_NEGOTIATE_NEEDED);  in recv_done()
633 		sc->status = SMBDIRECT_SOCKET_NEGOTIATE_RUNNING;  in recv_done()
634 		enqueue_reassembly(sc, recvmsg, 0);  in recv_done()
635 		wake_up(&sc->status_wait);  in recv_done()
645 			put_recvmsg(sc, recvmsg);  in recv_done()
646 			smb_direct_disconnect_rdma_connection(sc);  in recv_done()
655 			put_recvmsg(sc, recvmsg);  in recv_done()
656 			smb_direct_disconnect_rdma_connection(sc);  in recv_done()
663 			put_recvmsg(sc, recvmsg);  in recv_done()
664 			smb_direct_disconnect_rdma_connection(sc);  in recv_done()
669 			if (sc->recv_io.reassembly.full_packet_received)  in recv_done()
673 				sc->recv_io.reassembly.full_packet_received = false;  in recv_done()
675 				sc->recv_io.reassembly.full_packet_received = true;  in recv_done()
678 		atomic_dec(&sc->recv_io.posted.count);  in recv_done()
679 		atomic_dec(&sc->recv_io.credits.count);  in recv_done()
681 		old_recv_credit_target = sc->recv_io.credits.target;  in recv_done()
682 		sc->recv_io.credits.target =  in recv_done()
684 		sc->recv_io.credits.target =  in recv_done()
685 			min_t(u16, sc->recv_io.credits.target, sp->recv_credit_max);  in recv_done()
686 		sc->recv_io.credits.target =  in recv_done()
687 			max_t(u16, sc->recv_io.credits.target, 1);  in recv_done()
689 			   &sc->send_io.credits.count);  in recv_done()
693 			queue_work(sc->workqueue, &sc->idle.immediate_work);  in recv_done()
695 		if (atomic_read(&sc->send_io.credits.count) > 0)  in recv_done()
696 			wake_up(&sc->send_io.credits.wait_queue);  in recv_done()
699 			if (sc->recv_io.credits.target > old_recv_credit_target)  in recv_done()
700 				queue_work(sc->workqueue, &sc->recv_io.posted.refill_work);  in recv_done()
702 			enqueue_reassembly(sc, recvmsg, (int)data_length);  in recv_done()
703 			wake_up(&sc->recv_io.reassembly.wait_queue);  in recv_done()
705 			put_recvmsg(sc, recvmsg);  in recv_done()
717 	WARN_ON_ONCE(sc->recv_io.expected != SMBDIRECT_EXPECT_DATA_TRANSFER);  in recv_done()
718 	put_recvmsg(sc, recvmsg);  in recv_done()
719 	smb_direct_disconnect_rdma_connection(sc);  in recv_done()
722 static int smb_direct_post_recv(struct smbdirect_socket *sc,  in smb_direct_post_recv()  argument
725 	struct smbdirect_socket_parameters *sp = &sc->parameters;  in smb_direct_post_recv()
729 	recvmsg->sge.addr = ib_dma_map_single(sc->ib.dev,  in smb_direct_post_recv()
733 	ret = ib_dma_mapping_error(sc->ib.dev, recvmsg->sge.addr);  in smb_direct_post_recv()
737 	recvmsg->sge.lkey = sc->ib.pd->local_dma_lkey;  in smb_direct_post_recv()
745 	ret = ib_post_recv(sc->ib.qp, &wr, NULL);  in smb_direct_post_recv()
748 		ib_dma_unmap_single(sc->ib.dev,  in smb_direct_post_recv()
752 		smb_direct_disconnect_rdma_connection(sc);  in smb_direct_post_recv()
767 	struct smbdirect_socket *sc = &st->socket;  in smb_direct_read()  local
770 	if (sc->status != SMBDIRECT_SOCKET_CONNECTED) {  in smb_direct_read()
780 	if (sc->recv_io.reassembly.data_length >= size) {  in smb_direct_read()
793 		queue_length = sc->recv_io.reassembly.queue_length;  in smb_direct_read()
796 		offset = sc->recv_io.reassembly.first_entry_offset;  in smb_direct_read()
798 			recvmsg = get_first_reassembly(sc);  in smb_direct_read()
839 					spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags);  in smb_direct_read()
841 					spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags);  in smb_direct_read()
844 				put_recvmsg(sc, recvmsg);  in smb_direct_read()
854 		spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags);  in smb_direct_read()
855 		sc->recv_io.reassembly.data_length -= data_read;  in smb_direct_read()
856 		sc->recv_io.reassembly.queue_length -= queue_removed;  in smb_direct_read()
857 		spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags);  in smb_direct_read()
859 		sc->recv_io.reassembly.first_entry_offset = offset;  in smb_direct_read()
862 			    data_read, sc->recv_io.reassembly.data_length,  in smb_direct_read()
863 			    sc->recv_io.reassembly.first_entry_offset);  in smb_direct_read()
869 	rc = wait_event_interruptible(sc->recv_io.reassembly.wait_queue,  in smb_direct_read()
870 				      sc->recv_io.reassembly.data_length >= size ||  in smb_direct_read()
871 				       sc->status != SMBDIRECT_SOCKET_CONNECTED);  in smb_direct_read()
880 	struct smbdirect_socket *sc =  in smb_direct_post_recv_credits()  local
886 	if (atomic_read(&sc->recv_io.credits.count) < sc->recv_io.credits.target) {  in smb_direct_post_recv_credits()
888 			recvmsg = get_free_recvmsg(sc);  in smb_direct_post_recv_credits()
894 			ret = smb_direct_post_recv(sc, recvmsg);  in smb_direct_post_recv_credits()
897 				put_recvmsg(sc, recvmsg);  in smb_direct_post_recv_credits()
902 			atomic_inc(&sc->recv_io.posted.count);  in smb_direct_post_recv_credits()
907 		queue_work(sc->workqueue, &sc->idle.immediate_work);  in smb_direct_post_recv_credits()
913 	struct smbdirect_socket *sc;  in send_done()  local
917 	sc = sendmsg->socket;  in send_done()
927 		smb_direct_disconnect_rdma_connection(sc);  in send_done()
930 	if (atomic_dec_and_test(&sc->send_io.pending.count))  in send_done()
931 		wake_up(&sc->send_io.pending.zero_wait_queue);  in send_done()
939 		smb_direct_free_sendmsg(sc, sibling);  in send_done()
943 	smb_direct_free_sendmsg(sc, sibling);  in send_done()
946 static int manage_credits_prior_sending(struct smbdirect_socket *sc)  in manage_credits_prior_sending()  argument
950 	if (atomic_read(&sc->recv_io.credits.count) >= sc->recv_io.credits.target)  in manage_credits_prior_sending()
953 	new_credits = atomic_read(&sc->recv_io.posted.count);  in manage_credits_prior_sending()
957 	new_credits -= atomic_read(&sc->recv_io.credits.count);  in manage_credits_prior_sending()
961 	atomic_add(new_credits, &sc->recv_io.credits.count);  in manage_credits_prior_sending()
965 static int manage_keep_alive_before_sending(struct smbdirect_socket *sc)  in manage_keep_alive_before_sending()  argument
967 	struct smbdirect_socket_parameters *sp = &sc->parameters;  in manage_keep_alive_before_sending()
969 	if (sc->idle.keepalive == SMBDIRECT_KEEPALIVE_PENDING) {  in manage_keep_alive_before_sending()
970 		sc->idle.keepalive = SMBDIRECT_KEEPALIVE_SENT;  in manage_keep_alive_before_sending()
975 		mod_delayed_work(sc->workqueue, &sc->idle.timer_work,  in manage_keep_alive_before_sending()
982 static int smb_direct_post_send(struct smbdirect_socket *sc,  in smb_direct_post_send()  argument
987 	atomic_inc(&sc->send_io.pending.count);  in smb_direct_post_send()
988 	ret = ib_post_send(sc->ib.qp, wr, NULL);  in smb_direct_post_send()
991 		if (atomic_dec_and_test(&sc->send_io.pending.count))  in smb_direct_post_send()
992 			wake_up(&sc->send_io.pending.zero_wait_queue);  in smb_direct_post_send()
993 		smb_direct_disconnect_rdma_connection(sc);  in smb_direct_post_send()
1008 static int smb_direct_flush_send_list(struct smbdirect_socket *sc,  in smb_direct_flush_send_list()  argument
1035 	ret = smb_direct_post_send(sc, &first->wr);  in smb_direct_flush_send_list()
1041 		atomic_add(send_ctx->wr_cnt, &sc->send_io.credits.count);  in smb_direct_flush_send_list()
1042 		wake_up(&sc->send_io.credits.wait_queue);  in smb_direct_flush_send_list()
1045 			smb_direct_free_sendmsg(sc, first);  in smb_direct_flush_send_list()
1051 static int wait_for_credits(struct smbdirect_socket *sc,  in wait_for_credits()  argument
1064 					       sc->status != SMBDIRECT_SOCKET_CONNECTED);  in wait_for_credits()
1066 		if (sc->status != SMBDIRECT_SOCKET_CONNECTED)  in wait_for_credits()
1073 static int wait_for_send_credits(struct smbdirect_socket *sc,  in wait_for_send_credits()  argument
1079 	    (send_ctx->wr_cnt >= 16 || atomic_read(&sc->send_io.credits.count) <= 1)) {  in wait_for_send_credits()
1080 		ret = smb_direct_flush_send_list(sc, send_ctx, false);  in wait_for_send_credits()
1085 	return wait_for_credits(sc, &sc->send_io.credits.wait_queue, &sc->send_io.credits.count, 1);  in wait_for_send_credits()
1088 static int wait_for_rw_credits(struct smbdirect_socket *sc, int credits)  in wait_for_rw_credits()  argument
1090 	return wait_for_credits(sc,  in wait_for_rw_credits()
1091 				&sc->rw_io.credits.wait_queue,  in wait_for_rw_credits()
1092 				&sc->rw_io.credits.count,  in wait_for_rw_credits()
1096 static int calc_rw_credits(struct smbdirect_socket *sc,  in calc_rw_credits()  argument
1100 			    sc->rw_io.credits.num_pages);  in calc_rw_credits()
1103 static int smb_direct_create_header(struct smbdirect_socket *sc,  in smb_direct_create_header()  argument
1107 	struct smbdirect_socket_parameters *sp = &sc->parameters;  in smb_direct_create_header()
1113 	sendmsg = smb_direct_alloc_sendmsg(sc);  in smb_direct_create_header()
1120 	packet->credits_granted = cpu_to_le16(manage_credits_prior_sending(sc));  in smb_direct_create_header()
1123 	if (manage_keep_alive_before_sending(sc))  in smb_direct_create_header()
1150 	sendmsg->sge[0].addr = ib_dma_map_single(sc->ib.dev,  in smb_direct_create_header()
1154 	ret = ib_dma_mapping_error(sc->ib.dev, sendmsg->sge[0].addr);  in smb_direct_create_header()
1156 		smb_direct_free_sendmsg(sc, sendmsg);  in smb_direct_create_header()
1162 	sendmsg->sge[0].lkey = sc->ib.pd->local_dma_lkey;  in smb_direct_create_header()
1212 static int post_sendmsg(struct smbdirect_socket *sc,  in post_sendmsg()  argument
1219 		ib_dma_sync_single_for_device(sc->ib.dev,  in post_sendmsg()
1247 	return smb_direct_post_send(sc, &msg->wr);  in post_sendmsg()
1250 static int smb_direct_post_send_data(struct smbdirect_socket *sc,  in smb_direct_post_send_data()  argument
1260 	ret = wait_for_send_credits(sc, send_ctx);  in smb_direct_post_send_data()
1268 	ret = smb_direct_create_header(sc, data_length, remaining_data_length,  in smb_direct_post_send_data()
1271 		atomic_inc(&sc->send_io.credits.count);  in smb_direct_post_send_data()
1280 		sg_cnt = get_mapped_sg_list(sc->ib.dev,  in smb_direct_post_send_data()
1291 			ib_dma_unmap_sg(sc->ib.dev, sg, sg_cnt,  in smb_direct_post_send_data()
1300 			sge->lkey  = sc->ib.pd->local_dma_lkey;  in smb_direct_post_send_data()
1305 	ret = post_sendmsg(sc, send_ctx, msg);  in smb_direct_post_send_data()
1310 	smb_direct_free_sendmsg(sc, msg);  in smb_direct_post_send_data()
1311 	atomic_inc(&sc->send_io.credits.count);  in smb_direct_post_send_data()
1320 	struct smbdirect_socket *sc = &st->socket;  in smb_direct_writev()  local
1321 	struct smbdirect_socket_parameters *sp = &sc->parameters;  in smb_direct_writev()
1331 	if (sc->status != SMBDIRECT_SOCKET_CONNECTED)  in smb_direct_writev()
1433 		ret = smb_direct_post_send_data(sc, &send_ctx,  in smb_direct_writev()
1443 	ret = smb_direct_flush_send_list(sc, &send_ctx, true);  in smb_direct_writev()
1454 	wait_event(sc->send_io.pending.zero_wait_queue,  in smb_direct_writev()
1455 		   atomic_read(&sc->send_io.pending.count) == 0 ||  in smb_direct_writev()
1456 		   sc->status != SMBDIRECT_SOCKET_CONNECTED);  in smb_direct_writev()
1457 	if (sc->status != SMBDIRECT_SOCKET_CONNECTED && ret == 0)  in smb_direct_writev()
1467 	struct smbdirect_socket *sc = &t->socket;  in smb_direct_free_rdma_rw_msg()  local
1469 	rdma_rw_ctx_destroy(&msg->rdma_ctx, sc->ib.qp, sc->ib.qp->port,  in smb_direct_free_rdma_rw_msg()
1480 	struct smbdirect_socket *sc = msg->socket;  in read_write_done()  local
1487 			smb_direct_disconnect_rdma_connection(sc);  in read_write_done()
1509 	struct smbdirect_socket *sc = &t->socket;  in smb_direct_rdma_xmit()  local
1510 	struct smbdirect_socket_parameters *sp = &sc->parameters;  in smb_direct_rdma_xmit()
1520 	if (sc->status != SMBDIRECT_SOCKET_CONNECTED)  in smb_direct_rdma_xmit()
1543 		credits_needed += calc_rw_credits(sc, desc_buf, desc_buf_len);  in smb_direct_rdma_xmit()
1552 	ret = wait_for_rw_credits(sc, credits_needed);  in smb_direct_rdma_xmit()
1568 		msg->socket = sc;  in smb_direct_rdma_xmit()
1586 		ret = rdma_rw_ctx_init(&msg->rdma_ctx, sc->ib.qp, sc->ib.qp->port,  in smb_direct_rdma_xmit()
1605 		first_wr = rdma_rw_ctx_wrs(&msg->rdma_ctx, sc->ib.qp, sc->ib.qp->port,  in smb_direct_rdma_xmit()
1609 	ret = ib_post_send(sc->ib.qp, first_wr, NULL);  in smb_direct_rdma_xmit()
1624 	atomic_add(credits_needed, &sc->rw_io.credits.count);  in smb_direct_rdma_xmit()
1625 	wake_up(&sc->rw_io.credits.wait_queue);  in smb_direct_rdma_xmit()
1656 	struct smbdirect_socket *sc = &st->socket;  in smb_direct_disconnect()  local
1658 	ksmbd_debug(RDMA, "Disconnecting cm_id=%p\n", sc->rdma.cm_id);  in smb_direct_disconnect()
1666 	struct smbdirect_socket *sc = &st->socket;  in smb_direct_shutdown()  local
1668 	ksmbd_debug(RDMA, "smb-direct shutdown cm_id=%p\n", sc->rdma.cm_id);  in smb_direct_shutdown()
1670 	smb_direct_disconnect_rdma_work(&sc->disconnect_work);  in smb_direct_shutdown()
1676 	struct smbdirect_socket *sc = cm_id->context;  in smb_direct_cm_handler()  local
1683 		WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING);  in smb_direct_cm_handler()
1684 		sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED;  in smb_direct_cm_handler()
1685 		wake_up(&sc->status_wait);  in smb_direct_cm_handler()
1690 		ib_drain_qp(sc->ib.qp);  in smb_direct_cm_handler()
1692 		sc->status = SMBDIRECT_SOCKET_DISCONNECTED;  in smb_direct_cm_handler()
1693 		smb_direct_disconnect_rdma_work(&sc->disconnect_work);  in smb_direct_cm_handler()
1697 		sc->status = SMBDIRECT_SOCKET_DISCONNECTED;  in smb_direct_cm_handler()
1698 		smb_direct_disconnect_rdma_work(&sc->disconnect_work);  in smb_direct_cm_handler()
1712 	struct smbdirect_socket *sc = context;  in smb_direct_qpair_handler()  local
1715 		    sc->rdma.cm_id, ib_event_msg(event->event), event->event);  in smb_direct_qpair_handler()
1720 		smb_direct_disconnect_rdma_connection(sc);  in smb_direct_qpair_handler()
1727 static int smb_direct_send_negotiate_response(struct smbdirect_socket *sc,  in smb_direct_send_negotiate_response()  argument
1730 	struct smbdirect_socket_parameters *sp = &sc->parameters;  in smb_direct_send_negotiate_response()
1735 	sendmsg = smb_direct_alloc_sendmsg(sc);  in smb_direct_send_negotiate_response()
1746 		sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED;  in smb_direct_send_negotiate_response()
1755 		resp->credits_granted = cpu_to_le16(manage_credits_prior_sending(sc));  in smb_direct_send_negotiate_response()
1762 		sc->recv_io.expected = SMBDIRECT_EXPECT_DATA_TRANSFER;  in smb_direct_send_negotiate_response()
1763 		sc->status = SMBDIRECT_SOCKET_CONNECTED;  in smb_direct_send_negotiate_response()
1766 	sendmsg->sge[0].addr = ib_dma_map_single(sc->ib.dev,  in smb_direct_send_negotiate_response()
1769 	ret = ib_dma_mapping_error(sc->ib.dev, sendmsg->sge[0].addr);  in smb_direct_send_negotiate_response()
1771 		smb_direct_free_sendmsg(sc, sendmsg);  in smb_direct_send_negotiate_response()
1777 	sendmsg->sge[0].lkey = sc->ib.pd->local_dma_lkey;  in smb_direct_send_negotiate_response()
1779 	ret = post_sendmsg(sc, NULL, sendmsg);  in smb_direct_send_negotiate_response()
1781 		smb_direct_free_sendmsg(sc, sendmsg);  in smb_direct_send_negotiate_response()
1785 	wait_event(sc->send_io.pending.zero_wait_queue,  in smb_direct_send_negotiate_response()
1786 		   atomic_read(&sc->send_io.pending.count) == 0 ||  in smb_direct_send_negotiate_response()
1787 		   sc->status != SMBDIRECT_SOCKET_CONNECTED);  in smb_direct_send_negotiate_response()
1788 	if (sc->status != SMBDIRECT_SOCKET_CONNECTED)  in smb_direct_send_negotiate_response()
1794 static int smb_direct_accept_client(struct smbdirect_socket *sc)  in smb_direct_accept_client()  argument
1796 	struct smbdirect_socket_parameters *sp = &sc->parameters;  in smb_direct_accept_client()
1810 	if (sc->rdma.legacy_iwarp) {  in smb_direct_accept_client()
1827 	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING;  in smb_direct_accept_client()
1828 	mod_delayed_work(sc->workqueue, &sc->idle.timer_work,  in smb_direct_accept_client()
1831 	WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED);  in smb_direct_accept_client()
1832 	sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING;  in smb_direct_accept_client()
1833 	ret = rdma_accept(sc->rdma.cm_id, &conn_param);  in smb_direct_accept_client()
1841 static int smb_direct_prepare_negotiation(struct smbdirect_socket *sc)  in smb_direct_prepare_negotiation()  argument
1846 	WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED);  in smb_direct_prepare_negotiation()
1847 	sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED;  in smb_direct_prepare_negotiation()
1849 	sc->recv_io.expected = SMBDIRECT_EXPECT_NEGOTIATE_REQ;  in smb_direct_prepare_negotiation()
1851 	recvmsg = get_free_recvmsg(sc);  in smb_direct_prepare_negotiation()
1855 	ret = smb_direct_post_recv(sc, recvmsg);  in smb_direct_prepare_negotiation()
1861 	ret = smb_direct_accept_client(sc);  in smb_direct_prepare_negotiation()
1867 	smb_direct_post_recv_credits(&sc->recv_io.posted.refill_work);  in smb_direct_prepare_negotiation()
1870 	put_recvmsg(sc, recvmsg);  in smb_direct_prepare_negotiation()
1874 static unsigned int smb_direct_get_max_fr_pages(struct smbdirect_socket *sc)  in smb_direct_get_max_fr_pages()  argument
1877 		     sc->ib.dev->attrs.max_fast_reg_page_list_len,  in smb_direct_get_max_fr_pages()
1881 static int smb_direct_init_params(struct smbdirect_socket *sc,  in smb_direct_init_params()  argument
1884 	struct smbdirect_socket_parameters *sp = &sc->parameters;  in smb_direct_init_params()
1885 	struct ib_device *device = sc->ib.dev;  in smb_direct_init_params()
1905 	sc->rw_io.credits.num_pages = smb_direct_get_max_fr_pages(sc);  in smb_direct_init_params()
1906 	sc->rw_io.credits.max = DIV_ROUND_UP(sp->max_read_write_size,  in smb_direct_init_params()
1907 					 (sc->rw_io.credits.num_pages - 1) *  in smb_direct_init_params()
1915 			       DIV_ROUND_UP(sc->rw_io.credits.num_pages,  in smb_direct_init_params()
1917 	max_rw_wrs = sc->rw_io.credits.max * wrs_per_credit;  in smb_direct_init_params()
1949 	sc->recv_io.credits.target = 1;  in smb_direct_init_params()
1951 	atomic_set(&sc->rw_io.credits.count, sc->rw_io.credits.max);  in smb_direct_init_params()
1958 	cap->max_rdma_ctxs = sc->rw_io.credits.max;  in smb_direct_init_params()
1962 static void smb_direct_destroy_pools(struct smbdirect_socket *sc)  in smb_direct_destroy_pools()  argument
1966 	while ((recvmsg = get_free_recvmsg(sc)))  in smb_direct_destroy_pools()
1967 		mempool_free(recvmsg, sc->recv_io.mem.pool);  in smb_direct_destroy_pools()
1969 	mempool_destroy(sc->recv_io.mem.pool);  in smb_direct_destroy_pools()
1970 	sc->recv_io.mem.pool = NULL;  in smb_direct_destroy_pools()
1972 	kmem_cache_destroy(sc->recv_io.mem.cache);  in smb_direct_destroy_pools()
1973 	sc->recv_io.mem.cache = NULL;  in smb_direct_destroy_pools()
1975 	mempool_destroy(sc->send_io.mem.pool);  in smb_direct_destroy_pools()
1976 	sc->send_io.mem.pool = NULL;  in smb_direct_destroy_pools()
1978 	kmem_cache_destroy(sc->send_io.mem.cache);  in smb_direct_destroy_pools()
1979 	sc->send_io.mem.cache = NULL;  in smb_direct_destroy_pools()
1982 static int smb_direct_create_pools(struct smbdirect_socket *sc)  in smb_direct_create_pools()  argument
1984 	struct smbdirect_socket_parameters *sp = &sc->parameters;  in smb_direct_create_pools()
1989 	snprintf(name, sizeof(name), "smbdirect_send_io_pool_%p", sc);  in smb_direct_create_pools()
1990 	sc->send_io.mem.cache = kmem_cache_create(name,  in smb_direct_create_pools()
1994 	if (!sc->send_io.mem.cache)  in smb_direct_create_pools()
1997 	sc->send_io.mem.pool = mempool_create(sp->send_credit_target,  in smb_direct_create_pools()
1999 					    sc->send_io.mem.cache);  in smb_direct_create_pools()
2000 	if (!sc->send_io.mem.pool)  in smb_direct_create_pools()
2003 	snprintf(name, sizeof(name), "smbdirect_recv_io_pool_%p", sc);  in smb_direct_create_pools()
2004 	sc->recv_io.mem.cache = kmem_cache_create(name,  in smb_direct_create_pools()
2008 	if (!sc->recv_io.mem.cache)  in smb_direct_create_pools()
2011 	sc->recv_io.mem.pool =  in smb_direct_create_pools()
2013 			       mempool_free_slab, sc->recv_io.mem.cache);  in smb_direct_create_pools()
2014 	if (!sc->recv_io.mem.pool)  in smb_direct_create_pools()
2018 		recvmsg = mempool_alloc(sc->recv_io.mem.pool, KSMBD_DEFAULT_GFP);  in smb_direct_create_pools()
2021 		recvmsg->socket = sc;  in smb_direct_create_pools()
2023 		list_add(&recvmsg->list, &sc->recv_io.free.list);  in smb_direct_create_pools()
2028 	smb_direct_destroy_pools(sc);  in smb_direct_create_pools()
2032 static int smb_direct_create_qpair(struct smbdirect_socket *sc,  in smb_direct_create_qpair()  argument
2035 	struct smbdirect_socket_parameters *sp = &sc->parameters;  in smb_direct_create_qpair()
2040 	sc->ib.pd = ib_alloc_pd(sc->ib.dev, 0);  in smb_direct_create_qpair()
2041 	if (IS_ERR(sc->ib.pd)) {  in smb_direct_create_qpair()
2043 		ret = PTR_ERR(sc->ib.pd);  in smb_direct_create_qpair()
2044 		sc->ib.pd = NULL;  in smb_direct_create_qpair()
2048 	sc->ib.send_cq = ib_alloc_cq_any(sc->ib.dev, sc,  in smb_direct_create_qpair()
2052 	if (IS_ERR(sc->ib.send_cq)) {  in smb_direct_create_qpair()
2054 		ret = PTR_ERR(sc->ib.send_cq);  in smb_direct_create_qpair()
2055 		sc->ib.send_cq = NULL;  in smb_direct_create_qpair()
2059 	sc->ib.recv_cq = ib_alloc_cq_any(sc->ib.dev, sc,  in smb_direct_create_qpair()
2062 	if (IS_ERR(sc->ib.recv_cq)) {  in smb_direct_create_qpair()
2064 		ret = PTR_ERR(sc->ib.recv_cq);  in smb_direct_create_qpair()
2065 		sc->ib.recv_cq = NULL;  in smb_direct_create_qpair()
2071 	qp_attr.qp_context = sc;  in smb_direct_create_qpair()
2075 	qp_attr.send_cq = sc->ib.send_cq;  in smb_direct_create_qpair()
2076 	qp_attr.recv_cq = sc->ib.recv_cq;  in smb_direct_create_qpair()
2079 	ret = rdma_create_qp(sc->rdma.cm_id, sc->ib.pd, &qp_attr);  in smb_direct_create_qpair()
2085 	sc->ib.qp = sc->rdma.cm_id->qp;  in smb_direct_create_qpair()
2086 	sc->rdma.cm_id->event_handler = smb_direct_cm_handler;  in smb_direct_create_qpair()
2089 	if (pages_per_rw > sc->ib.dev->attrs.max_sgl_rd) {  in smb_direct_create_qpair()
2090 		ret = ib_mr_pool_init(sc->ib.qp, &sc->ib.qp->rdma_mrs,  in smb_direct_create_qpair()
2091 				      sc->rw_io.credits.max, IB_MR_TYPE_MEM_REG,  in smb_direct_create_qpair()
2092 				      sc->rw_io.credits.num_pages, 0);  in smb_direct_create_qpair()
2095 			       sc->rw_io.credits.max, sc->rw_io.credits.num_pages);  in smb_direct_create_qpair()
2102 	if (sc->ib.qp) {  in smb_direct_create_qpair()
2103 		sc->ib.qp = NULL;  in smb_direct_create_qpair()
2104 		rdma_destroy_qp(sc->rdma.cm_id);  in smb_direct_create_qpair()
2106 	if (sc->ib.recv_cq) {  in smb_direct_create_qpair()
2107 		ib_destroy_cq(sc->ib.recv_cq);  in smb_direct_create_qpair()
2108 		sc->ib.recv_cq = NULL;  in smb_direct_create_qpair()
2110 	if (sc->ib.send_cq) {  in smb_direct_create_qpair()
2111 		ib_destroy_cq(sc->ib.send_cq);  in smb_direct_create_qpair()
2112 		sc->ib.send_cq = NULL;  in smb_direct_create_qpair()
2114 	if (sc->ib.pd) {  in smb_direct_create_qpair()
2115 		ib_dealloc_pd(sc->ib.pd);  in smb_direct_create_qpair()
2116 		sc->ib.pd = NULL;  in smb_direct_create_qpair()
2124 	struct smbdirect_socket *sc = &st->socket;  in smb_direct_prepare()  local
2125 	struct smbdirect_socket_parameters *sp = &sc->parameters;  in smb_direct_prepare()
2144 	ret = wait_event_interruptible_timeout(sc->status_wait,  in smb_direct_prepare()
2145 					sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED &&  in smb_direct_prepare()
2146 					sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING &&  in smb_direct_prepare()
2147 					sc->status != SMBDIRECT_SOCKET_NEGOTIATE_NEEDED,  in smb_direct_prepare()
2149 	if (ret <= 0 || sc->status != SMBDIRECT_SOCKET_NEGOTIATE_RUNNING)  in smb_direct_prepare()
2152 	recvmsg = get_first_reassembly(sc);  in smb_direct_prepare()
2169 	sc->recv_io.credits.target = le16_to_cpu(req->credits_requested);  in smb_direct_prepare()
2170 	sc->recv_io.credits.target = min_t(u16, sc->recv_io.credits.target, sp->recv_credit_max);  in smb_direct_prepare()
2171 	sc->recv_io.credits.target = max_t(u16, sc->recv_io.credits.target, 1);  in smb_direct_prepare()
2173 	ret = smb_direct_send_negotiate_response(sc, ret);  in smb_direct_prepare()
2175 	spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags);  in smb_direct_prepare()
2176 	sc->recv_io.reassembly.queue_length--;  in smb_direct_prepare()
2178 	spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags);  in smb_direct_prepare()
2179 	put_recvmsg(sc, recvmsg);  in smb_direct_prepare()
2184 static int smb_direct_connect(struct smbdirect_socket *sc)  in smb_direct_connect()  argument
2189 	ret = smb_direct_init_params(sc, &qp_cap);  in smb_direct_connect()
2195 	ret = smb_direct_create_pools(sc);  in smb_direct_connect()
2201 	ret = smb_direct_create_qpair(sc, &qp_cap);  in smb_direct_connect()
2207 	ret = smb_direct_prepare_negotiation(sc);  in smb_direct_connect()
2228 	struct smbdirect_socket *sc;  in smb_direct_handle_connect_request()  local
2245 	sc = &t->socket;  in smb_direct_handle_connect_request()
2246 	sp = &sc->parameters;  in smb_direct_handle_connect_request()
2285 			sc->rdma.legacy_iwarp = true;  in smb_direct_handle_connect_request()
2309 	ret = smb_direct_connect(sc);  in smb_direct_handle_connect_request()