117 	struct smbdirect_socket *sc;  in get_smbd_max_read_write_size()  local
124 	sc = &t->socket;  in get_smbd_max_read_write_size()
125 	sp = &sc->parameters;  in get_smbd_max_read_write_size()
136 static void smb_direct_destroy_pools(struct smbdirect_socket *sc);
138 static int smb_direct_post_send_data(struct smbdirect_socket *sc,
150 smbdirect_recv_io *get_free_recvmsg(struct smbdirect_socket *sc)  in get_free_recvmsg()  argument
155 	spin_lock_irqsave(&sc->recv_io.free.lock, flags);  in get_free_recvmsg()
156 	if (!list_empty(&sc->recv_io.free.list)) {  in get_free_recvmsg()
157 		recvmsg = list_first_entry(&sc->recv_io.free.list,  in get_free_recvmsg()
162 	spin_unlock_irqrestore(&sc->recv_io.free.lock, flags);  in get_free_recvmsg()
166 static void put_recvmsg(struct smbdirect_socket *sc,  in put_recvmsg()  argument
172 		ib_dma_unmap_single(sc->ib.dev,  in put_recvmsg()
179 	spin_lock_irqsave(&sc->recv_io.free.lock, flags);  in put_recvmsg()
180 	list_add(&recvmsg->list, &sc->recv_io.free.list);  in put_recvmsg()
181 	spin_unlock_irqrestore(&sc->recv_io.free.lock, flags);  in put_recvmsg()
183 	queue_work(sc->workqueue, &sc->recv_io.posted.refill_work);  in put_recvmsg()
186 static void enqueue_reassembly(struct smbdirect_socket *sc,  in enqueue_reassembly()  argument
192 	spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags);  in enqueue_reassembly()
193 	list_add_tail(&recvmsg->list, &sc->recv_io.reassembly.list);  in enqueue_reassembly()
194 	sc->recv_io.reassembly.queue_length++;  in enqueue_reassembly()
202 	sc->recv_io.reassembly.data_length += data_length;  in enqueue_reassembly()
203 	spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags);  in enqueue_reassembly()
206 static struct smbdirect_recv_io *get_first_reassembly(struct smbdirect_socket *sc)  in get_first_reassembly()  argument
208 	if (!list_empty(&sc->recv_io.reassembly.list))  in get_first_reassembly()
209 		return list_first_entry(&sc->recv_io.reassembly.list,  in get_first_reassembly()
215 static void smb_direct_disconnect_wake_up_all(struct smbdirect_socket *sc)  in smb_direct_disconnect_wake_up_all()  argument
221 	wake_up_all(&sc->status_wait);  in smb_direct_disconnect_wake_up_all()
222 	wake_up_all(&sc->send_io.lcredits.wait_queue);  in smb_direct_disconnect_wake_up_all()
223 	wake_up_all(&sc->send_io.credits.wait_queue);  in smb_direct_disconnect_wake_up_all()
224 	wake_up_all(&sc->send_io.pending.zero_wait_queue);  in smb_direct_disconnect_wake_up_all()
225 	wake_up_all(&sc->recv_io.reassembly.wait_queue);  in smb_direct_disconnect_wake_up_all()
226 	wake_up_all(&sc->rw_io.credits.wait_queue);  in smb_direct_disconnect_wake_up_all()
231 	struct smbdirect_socket *sc =  in smb_direct_disconnect_rdma_work()  local
239 	disable_work(&sc->disconnect_work);  in smb_direct_disconnect_rdma_work()
240 	disable_work(&sc->recv_io.posted.refill_work);  in smb_direct_disconnect_rdma_work()
241 	disable_delayed_work(&sc->idle.timer_work);  in smb_direct_disconnect_rdma_work()
242 	disable_work(&sc->idle.immediate_work);  in smb_direct_disconnect_rdma_work()
244 	if (sc->first_error == 0)  in smb_direct_disconnect_rdma_work()
245 		sc->first_error = -ECONNABORTED;  in smb_direct_disconnect_rdma_work()
247 	switch (sc->status) {  in smb_direct_disconnect_rdma_work()
253 		sc->status = SMBDIRECT_SOCKET_DISCONNECTING;  in smb_direct_disconnect_rdma_work()
254 		rdma_disconnect(sc->rdma.cm_id);  in smb_direct_disconnect_rdma_work()
271 		sc->status = SMBDIRECT_SOCKET_DISCONNECTED;  in smb_direct_disconnect_rdma_work()
284 	smb_direct_disconnect_wake_up_all(sc);  in smb_direct_disconnect_rdma_work()
288 smb_direct_disconnect_rdma_connection(struct smbdirect_socket *sc)  in smb_direct_disconnect_rdma_connection()  argument
295 	disable_work(&sc->recv_io.posted.refill_work);  in smb_direct_disconnect_rdma_connection()
296 	disable_work(&sc->idle.immediate_work);  in smb_direct_disconnect_rdma_connection()
297 	disable_delayed_work(&sc->idle.timer_work);  in smb_direct_disconnect_rdma_connection()
299 	if (sc->first_error == 0)  in smb_direct_disconnect_rdma_connection()
300 		sc->first_error = -ECONNABORTED;  in smb_direct_disconnect_rdma_connection()
302 	switch (sc->status) {  in smb_direct_disconnect_rdma_connection()
318 		sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED;  in smb_direct_disconnect_rdma_connection()
323 		sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED;  in smb_direct_disconnect_rdma_connection()
328 		sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED;  in smb_direct_disconnect_rdma_connection()
333 		sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED;  in smb_direct_disconnect_rdma_connection()
337 		sc->status = SMBDIRECT_SOCKET_DISCONNECTED;  in smb_direct_disconnect_rdma_connection()
341 		sc->status = SMBDIRECT_SOCKET_ERROR;  in smb_direct_disconnect_rdma_connection()
349 	smb_direct_disconnect_wake_up_all(sc);  in smb_direct_disconnect_rdma_connection()
351 	queue_work(sc->workqueue, &sc->disconnect_work);  in smb_direct_disconnect_rdma_connection()
356 	struct smbdirect_socket *sc =  in smb_direct_send_immediate_work()  local
359 	if (sc->status != SMBDIRECT_SOCKET_CONNECTED)  in smb_direct_send_immediate_work()
362 	smb_direct_post_send_data(sc, NULL, NULL, 0, 0);  in smb_direct_send_immediate_work()
367 	struct smbdirect_socket *sc =  in smb_direct_idle_connection_timer()  local
369 	struct smbdirect_socket_parameters *sp = &sc->parameters;  in smb_direct_idle_connection_timer()
371 	if (sc->idle.keepalive != SMBDIRECT_KEEPALIVE_NONE) {  in smb_direct_idle_connection_timer()
372 		smb_direct_disconnect_rdma_connection(sc);  in smb_direct_idle_connection_timer()
376 	if (sc->status != SMBDIRECT_SOCKET_CONNECTED)  in smb_direct_idle_connection_timer()
383 	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING;  in smb_direct_idle_connection_timer()
384 	mod_delayed_work(sc->workqueue, &sc->idle.timer_work,  in smb_direct_idle_connection_timer()
386 	queue_work(sc->workqueue, &sc->idle.immediate_work);  in smb_direct_idle_connection_timer()
392 	struct smbdirect_socket *sc;  in alloc_transport()  local
399 	sc = &t->socket;  in alloc_transport()
400 	smbdirect_socket_init(sc);  in alloc_transport()
401 	sp = &sc->parameters;  in alloc_transport()
403 	sc->workqueue = smb_direct_wq;  in alloc_transport()
405 	INIT_WORK(&sc->disconnect_work, smb_direct_disconnect_rdma_work);  in alloc_transport()
419 	sc->rdma.cm_id = cm_id;  in alloc_transport()
420 	cm_id->context = sc;  in alloc_transport()
422 	sc->ib.dev = sc->rdma.cm_id->device;  in alloc_transport()
424 	INIT_DELAYED_WORK(&sc->idle.timer_work, smb_direct_idle_connection_timer);  in alloc_transport()
450 	struct smbdirect_socket *sc = &t->socket;  in free_transport()  local
453 	disable_work_sync(&sc->disconnect_work);  in free_transport()
454 	if (sc->status < SMBDIRECT_SOCKET_DISCONNECTING)  in free_transport()
455 		smb_direct_disconnect_rdma_work(&sc->disconnect_work);  in free_transport()
456 	if (sc->status < SMBDIRECT_SOCKET_DISCONNECTED)  in free_transport()
457 		wait_event(sc->status_wait, sc->status == SMBDIRECT_SOCKET_DISCONNECTED);  in free_transport()
466 	smb_direct_disconnect_wake_up_all(sc);  in free_transport()
468 	disable_work_sync(&sc->recv_io.posted.refill_work);  in free_transport()
469 	disable_delayed_work_sync(&sc->idle.timer_work);  in free_transport()
470 	disable_work_sync(&sc->idle.immediate_work);  in free_transport()
472 	if (sc->rdma.cm_id)  in free_transport()
473 		rdma_lock_handler(sc->rdma.cm_id);  in free_transport()
475 	if (sc->ib.qp) {  in free_transport()
476 		ib_drain_qp(sc->ib.qp);  in free_transport()
477 		sc->ib.qp = NULL;  in free_transport()
478 		rdma_destroy_qp(sc->rdma.cm_id);  in free_transport()
485 		spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags);  in free_transport()
486 		recvmsg = get_first_reassembly(sc);  in free_transport()
489 			spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags);  in free_transport()
490 			put_recvmsg(sc, recvmsg);  in free_transport()
492 			spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags);  in free_transport()
495 	sc->recv_io.reassembly.data_length = 0;  in free_transport()
497 	if (sc->ib.send_cq)  in free_transport()
498 		ib_free_cq(sc->ib.send_cq);  in free_transport()
499 	if (sc->ib.recv_cq)  in free_transport()
500 		ib_free_cq(sc->ib.recv_cq);  in free_transport()
501 	if (sc->ib.pd)  in free_transport()
502 		ib_dealloc_pd(sc->ib.pd);  in free_transport()
503 	if (sc->rdma.cm_id) {  in free_transport()
504 		rdma_unlock_handler(sc->rdma.cm_id);  in free_transport()
505 		rdma_destroy_id(sc->rdma.cm_id);  in free_transport()
508 	smb_direct_destroy_pools(sc);  in free_transport()
513 *smb_direct_alloc_sendmsg(struct smbdirect_socket *sc)  in smb_direct_alloc_sendmsg()  argument
517 	msg = mempool_alloc(sc->send_io.mem.pool, KSMBD_DEFAULT_GFP);  in smb_direct_alloc_sendmsg()
520 	msg->socket = sc;  in smb_direct_alloc_sendmsg()
526 static void smb_direct_free_sendmsg(struct smbdirect_socket *sc,  in smb_direct_free_sendmsg()  argument
538 		ib_dma_unmap_single(sc->ib.dev,  in smb_direct_free_sendmsg()
542 			ib_dma_unmap_page(sc->ib.dev,  in smb_direct_free_sendmsg()
546 	mempool_free(msg, sc->send_io.mem.pool);  in smb_direct_free_sendmsg()
551 	struct smbdirect_socket *sc = recvmsg->socket;  in smb_direct_check_recvmsg()  local
553 	switch (sc->recv_io.expected) {  in smb_direct_check_recvmsg()
601 	struct smbdirect_socket *sc;  in recv_done()  local
605 	sc = recvmsg->socket;  in recv_done()
606 	sp = &sc->parameters;  in recv_done()
609 		put_recvmsg(sc, recvmsg);  in recv_done()
614 			smb_direct_disconnect_rdma_connection(sc);  in recv_done()
630 	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE;  in recv_done()
631 	mod_delayed_work(sc->workqueue, &sc->idle.timer_work,  in recv_done()
634 	switch (sc->recv_io.expected) {  in recv_done()
637 			put_recvmsg(sc, recvmsg);  in recv_done()
638 			smb_direct_disconnect_rdma_connection(sc);  in recv_done()
641 		sc->recv_io.reassembly.full_packet_received = true;  in recv_done()
642 		WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_NEGOTIATE_NEEDED);  in recv_done()
643 		sc->status = SMBDIRECT_SOCKET_NEGOTIATE_RUNNING;  in recv_done()
644 		enqueue_reassembly(sc, recvmsg, 0);  in recv_done()
645 		wake_up(&sc->status_wait);  in recv_done()
655 			put_recvmsg(sc, recvmsg);  in recv_done()
656 			smb_direct_disconnect_rdma_connection(sc);  in recv_done()
665 			put_recvmsg(sc, recvmsg);  in recv_done()
666 			smb_direct_disconnect_rdma_connection(sc);  in recv_done()
673 			put_recvmsg(sc, recvmsg);  in recv_done()
674 			smb_direct_disconnect_rdma_connection(sc);  in recv_done()
679 			if (sc->recv_io.reassembly.full_packet_received)  in recv_done()
683 				sc->recv_io.reassembly.full_packet_received = false;  in recv_done()
685 				sc->recv_io.reassembly.full_packet_received = true;  in recv_done()
688 		atomic_dec(&sc->recv_io.posted.count);  in recv_done()
689 		atomic_dec(&sc->recv_io.credits.count);  in recv_done()
691 		old_recv_credit_target = sc->recv_io.credits.target;  in recv_done()
692 		sc->recv_io.credits.target =  in recv_done()
694 		sc->recv_io.credits.target =  in recv_done()
695 			min_t(u16, sc->recv_io.credits.target, sp->recv_credit_max);  in recv_done()
696 		sc->recv_io.credits.target =  in recv_done()
697 			max_t(u16, sc->recv_io.credits.target, 1);  in recv_done()
699 			   &sc->send_io.credits.count);  in recv_done()
703 			queue_work(sc->workqueue, &sc->idle.immediate_work);  in recv_done()
705 		if (atomic_read(&sc->send_io.credits.count) > 0)  in recv_done()
706 			wake_up(&sc->send_io.credits.wait_queue);  in recv_done()
709 			if (sc->recv_io.credits.target > old_recv_credit_target)  in recv_done()
710 				queue_work(sc->workqueue, &sc->recv_io.posted.refill_work);  in recv_done()
712 			enqueue_reassembly(sc, recvmsg, (int)data_length);  in recv_done()
713 			wake_up(&sc->recv_io.reassembly.wait_queue);  in recv_done()
715 			put_recvmsg(sc, recvmsg);  in recv_done()
727 	WARN_ON_ONCE(sc->recv_io.expected != SMBDIRECT_EXPECT_DATA_TRANSFER);  in recv_done()
728 	put_recvmsg(sc, recvmsg);  in recv_done()
729 	smb_direct_disconnect_rdma_connection(sc);  in recv_done()
732 static int smb_direct_post_recv(struct smbdirect_socket *sc,  in smb_direct_post_recv()  argument
735 	struct smbdirect_socket_parameters *sp = &sc->parameters;  in smb_direct_post_recv()
739 	recvmsg->sge.addr = ib_dma_map_single(sc->ib.dev,  in smb_direct_post_recv()
743 	ret = ib_dma_mapping_error(sc->ib.dev, recvmsg->sge.addr);  in smb_direct_post_recv()
747 	recvmsg->sge.lkey = sc->ib.pd->local_dma_lkey;  in smb_direct_post_recv()
755 	ret = ib_post_recv(sc->ib.qp, &wr, NULL);  in smb_direct_post_recv()
758 		ib_dma_unmap_single(sc->ib.dev,  in smb_direct_post_recv()
762 		smb_direct_disconnect_rdma_connection(sc);  in smb_direct_post_recv()
777 	struct smbdirect_socket *sc = &st->socket;  in smb_direct_read()  local
780 	if (sc->status != SMBDIRECT_SOCKET_CONNECTED) {  in smb_direct_read()
790 	if (sc->recv_io.reassembly.data_length >= size) {  in smb_direct_read()
803 		queue_length = sc->recv_io.reassembly.queue_length;  in smb_direct_read()
806 		offset = sc->recv_io.reassembly.first_entry_offset;  in smb_direct_read()
808 			recvmsg = get_first_reassembly(sc);  in smb_direct_read()
849 					spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags);  in smb_direct_read()
851 					spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags);  in smb_direct_read()
854 				put_recvmsg(sc, recvmsg);  in smb_direct_read()
864 		spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags);  in smb_direct_read()
865 		sc->recv_io.reassembly.data_length -= data_read;  in smb_direct_read()
866 		sc->recv_io.reassembly.queue_length -= queue_removed;  in smb_direct_read()
867 		spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags);  in smb_direct_read()
869 		sc->recv_io.reassembly.first_entry_offset = offset;  in smb_direct_read()
872 			    data_read, sc->recv_io.reassembly.data_length,  in smb_direct_read()
873 			    sc->recv_io.reassembly.first_entry_offset);  in smb_direct_read()
879 	rc = wait_event_interruptible(sc->recv_io.reassembly.wait_queue,  in smb_direct_read()
880 				      sc->recv_io.reassembly.data_length >= size ||  in smb_direct_read()
881 				       sc->status != SMBDIRECT_SOCKET_CONNECTED);  in smb_direct_read()
890 	struct smbdirect_socket *sc =  in smb_direct_post_recv_credits()  local
896 	if (atomic_read(&sc->recv_io.credits.count) < sc->recv_io.credits.target) {  in smb_direct_post_recv_credits()
898 			recvmsg = get_free_recvmsg(sc);  in smb_direct_post_recv_credits()
904 			ret = smb_direct_post_recv(sc, recvmsg);  in smb_direct_post_recv_credits()
907 				put_recvmsg(sc, recvmsg);  in smb_direct_post_recv_credits()
912 			atomic_inc(&sc->recv_io.posted.count);  in smb_direct_post_recv_credits()
917 		queue_work(sc->workqueue, &sc->idle.immediate_work);  in smb_direct_post_recv_credits()
923 	struct smbdirect_socket *sc;  in send_done()  local
927 	sc = sendmsg->socket;  in send_done()
938 		smb_direct_free_sendmsg(sc, sibling);  in send_done()
942 	smb_direct_free_sendmsg(sc, sendmsg);  in send_done()
949 		smb_direct_disconnect_rdma_connection(sc);  in send_done()
953 	atomic_add(lcredits, &sc->send_io.lcredits.count);  in send_done()
954 	wake_up(&sc->send_io.lcredits.wait_queue);  in send_done()
956 	if (atomic_dec_and_test(&sc->send_io.pending.count))  in send_done()
957 		wake_up(&sc->send_io.pending.zero_wait_queue);  in send_done()
960 static int manage_credits_prior_sending(struct smbdirect_socket *sc)  in manage_credits_prior_sending()  argument
964 	if (atomic_read(&sc->recv_io.credits.count) >= sc->recv_io.credits.target)  in manage_credits_prior_sending()
967 	new_credits = atomic_read(&sc->recv_io.posted.count);  in manage_credits_prior_sending()
971 	new_credits -= atomic_read(&sc->recv_io.credits.count);  in manage_credits_prior_sending()
975 	atomic_add(new_credits, &sc->recv_io.credits.count);  in manage_credits_prior_sending()
979 static int manage_keep_alive_before_sending(struct smbdirect_socket *sc)  in manage_keep_alive_before_sending()  argument
981 	struct smbdirect_socket_parameters *sp = &sc->parameters;  in manage_keep_alive_before_sending()
983 	if (sc->idle.keepalive == SMBDIRECT_KEEPALIVE_PENDING) {  in manage_keep_alive_before_sending()
984 		sc->idle.keepalive = SMBDIRECT_KEEPALIVE_SENT;  in manage_keep_alive_before_sending()
989 		mod_delayed_work(sc->workqueue, &sc->idle.timer_work,  in manage_keep_alive_before_sending()
996 static int smb_direct_post_send(struct smbdirect_socket *sc,  in smb_direct_post_send()  argument
1001 	atomic_inc(&sc->send_io.pending.count);  in smb_direct_post_send()
1002 	ret = ib_post_send(sc->ib.qp, wr, NULL);  in smb_direct_post_send()
1005 		smb_direct_disconnect_rdma_connection(sc);  in smb_direct_post_send()
1020 static int smb_direct_flush_send_list(struct smbdirect_socket *sc,  in smb_direct_flush_send_list()  argument
1059 	ret = smb_direct_post_send(sc, &first->wr);  in smb_direct_flush_send_list()
1065 			smb_direct_free_sendmsg(sc, sibling);  in smb_direct_flush_send_list()
1067 		smb_direct_free_sendmsg(sc, last);  in smb_direct_flush_send_list()
1073 static int wait_for_credits(struct smbdirect_socket *sc,  in wait_for_credits()  argument
1086 					       sc->status != SMBDIRECT_SOCKET_CONNECTED);  in wait_for_credits()
1088 		if (sc->status != SMBDIRECT_SOCKET_CONNECTED)  in wait_for_credits()
1095 static int wait_for_send_lcredit(struct smbdirect_socket *sc,  in wait_for_send_lcredit()  argument
1098 	if (send_ctx && (atomic_read(&sc->send_io.lcredits.count) <= 1)) {  in wait_for_send_lcredit()
1101 		ret = smb_direct_flush_send_list(sc, send_ctx, false);  in wait_for_send_lcredit()
1106 	return wait_for_credits(sc,  in wait_for_send_lcredit()
1107 				&sc->send_io.lcredits.wait_queue,  in wait_for_send_lcredit()
1108 				&sc->send_io.lcredits.count,  in wait_for_send_lcredit()
1112 static int wait_for_send_credits(struct smbdirect_socket *sc,  in wait_for_send_credits()  argument
1118 	    (send_ctx->wr_cnt >= 16 || atomic_read(&sc->send_io.credits.count) <= 1)) {  in wait_for_send_credits()
1119 		ret = smb_direct_flush_send_list(sc, send_ctx, false);  in wait_for_send_credits()
1124 	return wait_for_credits(sc, &sc->send_io.credits.wait_queue, &sc->send_io.credits.count, 1);  in wait_for_send_credits()
1127 static int wait_for_rw_credits(struct smbdirect_socket *sc, int credits)  in wait_for_rw_credits()  argument
1129 	return wait_for_credits(sc,  in wait_for_rw_credits()
1130 				&sc->rw_io.credits.wait_queue,  in wait_for_rw_credits()
1131 				&sc->rw_io.credits.count,  in wait_for_rw_credits()
1135 static int calc_rw_credits(struct smbdirect_socket *sc,  in calc_rw_credits()  argument
1139 			    sc->rw_io.credits.num_pages);  in calc_rw_credits()
1142 static int smb_direct_create_header(struct smbdirect_socket *sc,  in smb_direct_create_header()  argument
1146 	struct smbdirect_socket_parameters *sp = &sc->parameters;  in smb_direct_create_header()
1152 	sendmsg = smb_direct_alloc_sendmsg(sc);  in smb_direct_create_header()
1159 	packet->credits_granted = cpu_to_le16(manage_credits_prior_sending(sc));  in smb_direct_create_header()
1162 	if (manage_keep_alive_before_sending(sc))  in smb_direct_create_header()
1189 	sendmsg->sge[0].addr = ib_dma_map_single(sc->ib.dev,  in smb_direct_create_header()
1193 	ret = ib_dma_mapping_error(sc->ib.dev, sendmsg->sge[0].addr);  in smb_direct_create_header()
1195 		smb_direct_free_sendmsg(sc, sendmsg);  in smb_direct_create_header()
1201 	sendmsg->sge[0].lkey = sc->ib.pd->local_dma_lkey;  in smb_direct_create_header()
1251 static int post_sendmsg(struct smbdirect_socket *sc,  in post_sendmsg()  argument
1258 		ib_dma_sync_single_for_device(sc->ib.dev,  in post_sendmsg()
1286 	return smb_direct_post_send(sc, &msg->wr);  in post_sendmsg()
1289 static int smb_direct_post_send_data(struct smbdirect_socket *sc,  in smb_direct_post_send_data()  argument
1299 	ret = wait_for_send_lcredit(sc, send_ctx);  in smb_direct_post_send_data()
1303 	ret = wait_for_send_credits(sc, send_ctx);  in smb_direct_post_send_data()
1311 	ret = smb_direct_create_header(sc, data_length, remaining_data_length,  in smb_direct_post_send_data()
1321 		sg_cnt = get_mapped_sg_list(sc->ib.dev,  in smb_direct_post_send_data()
1332 			ib_dma_unmap_sg(sc->ib.dev, sg, sg_cnt,  in smb_direct_post_send_data()
1341 			sge->lkey  = sc->ib.pd->local_dma_lkey;  in smb_direct_post_send_data()
1346 	ret = post_sendmsg(sc, send_ctx, msg);  in smb_direct_post_send_data()
1351 	smb_direct_free_sendmsg(sc, msg);  in smb_direct_post_send_data()
1353 	atomic_inc(&sc->send_io.credits.count);  in smb_direct_post_send_data()
1355 	atomic_inc(&sc->send_io.lcredits.count);  in smb_direct_post_send_data()
1365 	struct smbdirect_socket *sc = &st->socket;  in smb_direct_writev()  local
1366 	struct smbdirect_socket_parameters *sp = &sc->parameters;  in smb_direct_writev()
1376 	if (sc->status != SMBDIRECT_SOCKET_CONNECTED)  in smb_direct_writev()
1478 		ret = smb_direct_post_send_data(sc, &send_ctx,  in smb_direct_writev()
1488 	ret = smb_direct_flush_send_list(sc, &send_ctx, true);  in smb_direct_writev()
1499 	wait_event(sc->send_io.pending.zero_wait_queue,  in smb_direct_writev()
1500 		   atomic_read(&sc->send_io.pending.count) == 0 ||  in smb_direct_writev()
1501 		   sc->status != SMBDIRECT_SOCKET_CONNECTED);  in smb_direct_writev()
1502 	if (sc->status != SMBDIRECT_SOCKET_CONNECTED && ret == 0)  in smb_direct_writev()
1512 	struct smbdirect_socket *sc = &t->socket;  in smb_direct_free_rdma_rw_msg()  local
1514 	rdma_rw_ctx_destroy(&msg->rdma_ctx, sc->ib.qp, sc->ib.qp->port,  in smb_direct_free_rdma_rw_msg()
1525 	struct smbdirect_socket *sc = msg->socket;  in read_write_done()  local
1532 			smb_direct_disconnect_rdma_connection(sc);  in read_write_done()
1554 	struct smbdirect_socket *sc = &t->socket;  in smb_direct_rdma_xmit()  local
1555 	struct smbdirect_socket_parameters *sp = &sc->parameters;  in smb_direct_rdma_xmit()
1565 	if (sc->status != SMBDIRECT_SOCKET_CONNECTED)  in smb_direct_rdma_xmit()
1588 		credits_needed += calc_rw_credits(sc, desc_buf, desc_buf_len);  in smb_direct_rdma_xmit()
1597 	ret = wait_for_rw_credits(sc, credits_needed);  in smb_direct_rdma_xmit()
1613 		msg->socket = sc;  in smb_direct_rdma_xmit()
1631 		ret = rdma_rw_ctx_init(&msg->rdma_ctx, sc->ib.qp, sc->ib.qp->port,  in smb_direct_rdma_xmit()
1650 		first_wr = rdma_rw_ctx_wrs(&msg->rdma_ctx, sc->ib.qp, sc->ib.qp->port,  in smb_direct_rdma_xmit()
1654 	ret = ib_post_send(sc->ib.qp, first_wr, NULL);  in smb_direct_rdma_xmit()
1669 	atomic_add(credits_needed, &sc->rw_io.credits.count);  in smb_direct_rdma_xmit()
1670 	wake_up(&sc->rw_io.credits.wait_queue);  in smb_direct_rdma_xmit()
1701 	struct smbdirect_socket *sc = &st->socket;  in smb_direct_disconnect()  local
1703 	ksmbd_debug(RDMA, "Disconnecting cm_id=%p\n", sc->rdma.cm_id);  in smb_direct_disconnect()
1711 	struct smbdirect_socket *sc = &st->socket;  in smb_direct_shutdown()  local
1713 	ksmbd_debug(RDMA, "smb-direct shutdown cm_id=%p\n", sc->rdma.cm_id);  in smb_direct_shutdown()
1715 	smb_direct_disconnect_rdma_work(&sc->disconnect_work);  in smb_direct_shutdown()
1721 	struct smbdirect_socket *sc = cm_id->context;  in smb_direct_cm_handler()  local
1728 		WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING);  in smb_direct_cm_handler()
1729 		sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED;  in smb_direct_cm_handler()
1730 		wake_up(&sc->status_wait);  in smb_direct_cm_handler()
1735 		sc->status = SMBDIRECT_SOCKET_DISCONNECTED;  in smb_direct_cm_handler()
1736 		smb_direct_disconnect_rdma_work(&sc->disconnect_work);  in smb_direct_cm_handler()
1737 		if (sc->ib.qp)  in smb_direct_cm_handler()
1738 			ib_drain_qp(sc->ib.qp);  in smb_direct_cm_handler()
1742 		sc->status = SMBDIRECT_SOCKET_DISCONNECTED;  in smb_direct_cm_handler()
1743 		smb_direct_disconnect_rdma_work(&sc->disconnect_work);  in smb_direct_cm_handler()
1757 	struct smbdirect_socket *sc = context;  in smb_direct_qpair_handler()  local
1760 		    sc->rdma.cm_id, ib_event_msg(event->event), event->event);  in smb_direct_qpair_handler()
1765 		smb_direct_disconnect_rdma_connection(sc);  in smb_direct_qpair_handler()
1772 static int smb_direct_send_negotiate_response(struct smbdirect_socket *sc,  in smb_direct_send_negotiate_response()  argument
1775 	struct smbdirect_socket_parameters *sp = &sc->parameters;  in smb_direct_send_negotiate_response()
1780 	sendmsg = smb_direct_alloc_sendmsg(sc);  in smb_direct_send_negotiate_response()
1791 		sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED;  in smb_direct_send_negotiate_response()
1800 		resp->credits_granted = cpu_to_le16(manage_credits_prior_sending(sc));  in smb_direct_send_negotiate_response()
1807 		sc->recv_io.expected = SMBDIRECT_EXPECT_DATA_TRANSFER;  in smb_direct_send_negotiate_response()
1808 		sc->status = SMBDIRECT_SOCKET_CONNECTED;  in smb_direct_send_negotiate_response()
1811 	sendmsg->sge[0].addr = ib_dma_map_single(sc->ib.dev,  in smb_direct_send_negotiate_response()
1814 	ret = ib_dma_mapping_error(sc->ib.dev, sendmsg->sge[0].addr);  in smb_direct_send_negotiate_response()
1816 		smb_direct_free_sendmsg(sc, sendmsg);  in smb_direct_send_negotiate_response()
1822 	sendmsg->sge[0].lkey = sc->ib.pd->local_dma_lkey;  in smb_direct_send_negotiate_response()
1824 	ret = post_sendmsg(sc, NULL, sendmsg);  in smb_direct_send_negotiate_response()
1826 		smb_direct_free_sendmsg(sc, sendmsg);  in smb_direct_send_negotiate_response()
1830 	wait_event(sc->send_io.pending.zero_wait_queue,  in smb_direct_send_negotiate_response()
1831 		   atomic_read(&sc->send_io.pending.count) == 0 ||  in smb_direct_send_negotiate_response()
1832 		   sc->status != SMBDIRECT_SOCKET_CONNECTED);  in smb_direct_send_negotiate_response()
1833 	if (sc->status != SMBDIRECT_SOCKET_CONNECTED)  in smb_direct_send_negotiate_response()
1839 static int smb_direct_accept_client(struct smbdirect_socket *sc)  in smb_direct_accept_client()  argument
1841 	struct smbdirect_socket_parameters *sp = &sc->parameters;  in smb_direct_accept_client()
1855 	if (sc->rdma.legacy_iwarp) {  in smb_direct_accept_client()
1872 	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING;  in smb_direct_accept_client()
1873 	mod_delayed_work(sc->workqueue, &sc->idle.timer_work,  in smb_direct_accept_client()
1876 	WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED);  in smb_direct_accept_client()
1877 	sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING;  in smb_direct_accept_client()
1878 	ret = rdma_accept(sc->rdma.cm_id, &conn_param);  in smb_direct_accept_client()
1886 static int smb_direct_prepare_negotiation(struct smbdirect_socket *sc)  in smb_direct_prepare_negotiation()  argument
1892 	WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED);  in smb_direct_prepare_negotiation()
1893 	sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED;  in smb_direct_prepare_negotiation()
1895 	sc->recv_io.expected = SMBDIRECT_EXPECT_NEGOTIATE_REQ;  in smb_direct_prepare_negotiation()
1897 	recvmsg = get_free_recvmsg(sc);  in smb_direct_prepare_negotiation()
1901 	ret = smb_direct_post_recv(sc, recvmsg);  in smb_direct_prepare_negotiation()
1908 	ret = smb_direct_accept_client(sc);  in smb_direct_prepare_negotiation()
1923 		put_recvmsg(sc, recvmsg);  in smb_direct_prepare_negotiation()
1927 static int smb_direct_init_params(struct smbdirect_socket *sc)  in smb_direct_init_params()  argument
1929 	struct smbdirect_socket_parameters *sp = &sc->parameters;  in smb_direct_init_params()
1942 	atomic_set(&sc->send_io.lcredits.count, sp->send_credit_target);  in smb_direct_init_params()
1945 	sc->rw_io.credits.max = rdma_rw_mr_factor(sc->ib.dev,  in smb_direct_init_params()
1946 						  sc->rdma.cm_id->port_num,  in smb_direct_init_params()
1948 	sc->rw_io.credits.num_pages = DIV_ROUND_UP(maxpages, sc->rw_io.credits.max);  in smb_direct_init_params()
1950 	sc->rw_io.credits.max += 1;  in smb_direct_init_params()
1952 	sc->recv_io.credits.target = 1;  in smb_direct_init_params()
1954 	atomic_set(&sc->rw_io.credits.count, sc->rw_io.credits.max);  in smb_direct_init_params()
1959 static void smb_direct_destroy_pools(struct smbdirect_socket *sc)  in smb_direct_destroy_pools()  argument
1963 	while ((recvmsg = get_free_recvmsg(sc)))  in smb_direct_destroy_pools()
1964 		mempool_free(recvmsg, sc->recv_io.mem.pool);  in smb_direct_destroy_pools()
1966 	mempool_destroy(sc->recv_io.mem.pool);  in smb_direct_destroy_pools()
1967 	sc->recv_io.mem.pool = NULL;  in smb_direct_destroy_pools()
1969 	kmem_cache_destroy(sc->recv_io.mem.cache);  in smb_direct_destroy_pools()
1970 	sc->recv_io.mem.cache = NULL;  in smb_direct_destroy_pools()
1972 	mempool_destroy(sc->send_io.mem.pool);  in smb_direct_destroy_pools()
1973 	sc->send_io.mem.pool = NULL;  in smb_direct_destroy_pools()
1975 	kmem_cache_destroy(sc->send_io.mem.cache);  in smb_direct_destroy_pools()
1976 	sc->send_io.mem.cache = NULL;  in smb_direct_destroy_pools()
1979 static int smb_direct_create_pools(struct smbdirect_socket *sc)  in smb_direct_create_pools()  argument
1981 	struct smbdirect_socket_parameters *sp = &sc->parameters;  in smb_direct_create_pools()
1986 	snprintf(name, sizeof(name), "smbdirect_send_io_pool_%p", sc);  in smb_direct_create_pools()
1987 	sc->send_io.mem.cache = kmem_cache_create(name,  in smb_direct_create_pools()
1991 	if (!sc->send_io.mem.cache)  in smb_direct_create_pools()
1994 	sc->send_io.mem.pool = mempool_create(sp->send_credit_target,  in smb_direct_create_pools()
1996 					    sc->send_io.mem.cache);  in smb_direct_create_pools()
1997 	if (!sc->send_io.mem.pool)  in smb_direct_create_pools()
2000 	snprintf(name, sizeof(name), "smbdirect_recv_io_pool_%p", sc);  in smb_direct_create_pools()
2001 	sc->recv_io.mem.cache = kmem_cache_create(name,  in smb_direct_create_pools()
2005 	if (!sc->recv_io.mem.cache)  in smb_direct_create_pools()
2008 	sc->recv_io.mem.pool =  in smb_direct_create_pools()
2010 			       mempool_free_slab, sc->recv_io.mem.cache);  in smb_direct_create_pools()
2011 	if (!sc->recv_io.mem.pool)  in smb_direct_create_pools()
2015 		recvmsg = mempool_alloc(sc->recv_io.mem.pool, KSMBD_DEFAULT_GFP);  in smb_direct_create_pools()
2018 		recvmsg->socket = sc;  in smb_direct_create_pools()
2020 		list_add(&recvmsg->list, &sc->recv_io.free.list);  in smb_direct_create_pools()
2025 	smb_direct_destroy_pools(sc);  in smb_direct_create_pools()
2061 static int smb_direct_create_qpair(struct smbdirect_socket *sc)  in smb_direct_create_qpair()  argument
2063 	struct smbdirect_socket_parameters *sp = &sc->parameters;  in smb_direct_create_qpair()
2084 	qp_cap.max_rdma_ctxs = sc->rw_io.credits.max;  in smb_direct_create_qpair()
2096 	qp_attr.port_num = sc->rdma.cm_id->port_num;  in smb_direct_create_qpair()
2097 	rdma_send_wr = smb_direct_rdma_rw_send_wrs(sc->ib.dev, &qp_attr);  in smb_direct_create_qpair()
2100 	if (qp_cap.max_send_wr > sc->ib.dev->attrs.max_cqe ||  in smb_direct_create_qpair()
2101 	    qp_cap.max_send_wr > sc->ib.dev->attrs.max_qp_wr) {  in smb_direct_create_qpair()
2106 		       sc->ib.dev->name,  in smb_direct_create_qpair()
2107 		       sc->ib.dev->attrs.max_cqe,  in smb_direct_create_qpair()
2108 		       sc->ib.dev->attrs.max_qp_wr);  in smb_direct_create_qpair()
2115 	    (max_send_wr >= sc->ib.dev->attrs.max_cqe ||  in smb_direct_create_qpair()
2116 	     max_send_wr >= sc->ib.dev->attrs.max_qp_wr)) {  in smb_direct_create_qpair()
2121 		       sc->ib.dev->name,  in smb_direct_create_qpair()
2122 		       sc->ib.dev->attrs.max_cqe,  in smb_direct_create_qpair()
2123 		       sc->ib.dev->attrs.max_qp_wr);  in smb_direct_create_qpair()
2129 	if (qp_cap.max_recv_wr > sc->ib.dev->attrs.max_cqe ||  in smb_direct_create_qpair()
2130 	    qp_cap.max_recv_wr > sc->ib.dev->attrs.max_qp_wr) {  in smb_direct_create_qpair()
2135 		       sc->ib.dev->name,  in smb_direct_create_qpair()
2136 		       sc->ib.dev->attrs.max_cqe,  in smb_direct_create_qpair()
2137 		       sc->ib.dev->attrs.max_qp_wr);  in smb_direct_create_qpair()
2143 	if (qp_cap.max_send_sge > sc->ib.dev->attrs.max_send_sge ||  in smb_direct_create_qpair()
2144 	    qp_cap.max_recv_sge > sc->ib.dev->attrs.max_recv_sge) {  in smb_direct_create_qpair()
2147 		       sc->ib.dev->name,  in smb_direct_create_qpair()
2148 		       sc->ib.dev->attrs.max_send_sge,  in smb_direct_create_qpair()
2149 		       sc->ib.dev->attrs.max_recv_sge);  in smb_direct_create_qpair()
2153 	sc->ib.pd = ib_alloc_pd(sc->ib.dev, 0);  in smb_direct_create_qpair()
2154 	if (IS_ERR(sc->ib.pd)) {  in smb_direct_create_qpair()
2156 		ret = PTR_ERR(sc->ib.pd);  in smb_direct_create_qpair()
2157 		sc->ib.pd = NULL;  in smb_direct_create_qpair()
2161 	sc->ib.send_cq = ib_alloc_cq_any(sc->ib.dev, sc,  in smb_direct_create_qpair()
2164 	if (IS_ERR(sc->ib.send_cq)) {  in smb_direct_create_qpair()
2166 		ret = PTR_ERR(sc->ib.send_cq);  in smb_direct_create_qpair()
2167 		sc->ib.send_cq = NULL;  in smb_direct_create_qpair()
2171 	sc->ib.recv_cq = ib_alloc_cq_any(sc->ib.dev, sc,  in smb_direct_create_qpair()
2174 	if (IS_ERR(sc->ib.recv_cq)) {  in smb_direct_create_qpair()
2176 		ret = PTR_ERR(sc->ib.recv_cq);  in smb_direct_create_qpair()
2177 		sc->ib.recv_cq = NULL;  in smb_direct_create_qpair()
2191 	qp_attr.qp_context = sc;  in smb_direct_create_qpair()
2195 	qp_attr.send_cq = sc->ib.send_cq;  in smb_direct_create_qpair()
2196 	qp_attr.recv_cq = sc->ib.recv_cq;  in smb_direct_create_qpair()
2199 	ret = rdma_create_qp(sc->rdma.cm_id, sc->ib.pd, &qp_attr);  in smb_direct_create_qpair()
2205 	sc->ib.qp = sc->rdma.cm_id->qp;  in smb_direct_create_qpair()
2206 	sc->rdma.cm_id->event_handler = smb_direct_cm_handler;  in smb_direct_create_qpair()
2210 	if (sc->ib.qp) {  in smb_direct_create_qpair()
2211 		sc->ib.qp = NULL;  in smb_direct_create_qpair()
2212 		rdma_destroy_qp(sc->rdma.cm_id);  in smb_direct_create_qpair()
2214 	if (sc->ib.recv_cq) {  in smb_direct_create_qpair()
2215 		ib_destroy_cq(sc->ib.recv_cq);  in smb_direct_create_qpair()
2216 		sc->ib.recv_cq = NULL;  in smb_direct_create_qpair()
2218 	if (sc->ib.send_cq) {  in smb_direct_create_qpair()
2219 		ib_destroy_cq(sc->ib.send_cq);  in smb_direct_create_qpair()
2220 		sc->ib.send_cq = NULL;  in smb_direct_create_qpair()
2222 	if (sc->ib.pd) {  in smb_direct_create_qpair()
2223 		ib_dealloc_pd(sc->ib.pd);  in smb_direct_create_qpair()
2224 		sc->ib.pd = NULL;  in smb_direct_create_qpair()
2232 	struct smbdirect_socket *sc = &st->socket;  in smb_direct_prepare()  local
2233 	struct smbdirect_socket_parameters *sp = &sc->parameters;  in smb_direct_prepare()
2252 	ret = wait_event_interruptible_timeout(sc->status_wait,  in smb_direct_prepare()
2253 					sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED &&  in smb_direct_prepare()
2254 					sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING &&  in smb_direct_prepare()
2255 					sc->status != SMBDIRECT_SOCKET_NEGOTIATE_NEEDED,  in smb_direct_prepare()
2257 	if (ret <= 0 || sc->status != SMBDIRECT_SOCKET_NEGOTIATE_RUNNING)  in smb_direct_prepare()
2260 	recvmsg = get_first_reassembly(sc);  in smb_direct_prepare()
2277 	sc->recv_io.credits.target = le16_to_cpu(req->credits_requested);  in smb_direct_prepare()
2278 	sc->recv_io.credits.target = min_t(u16, sc->recv_io.credits.target, sp->recv_credit_max);  in smb_direct_prepare()
2279 	sc->recv_io.credits.target = max_t(u16, sc->recv_io.credits.target, 1);  in smb_direct_prepare()
2282 	spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags);  in smb_direct_prepare()
2283 	sc->recv_io.reassembly.queue_length--;  in smb_direct_prepare()
2285 	spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags);  in smb_direct_prepare()
2286 	put_recvmsg(sc, recvmsg);  in smb_direct_prepare()
2296 	 * We do that with sc->idle.immediate_work still being disabled  in smb_direct_prepare()
2297 	 * via smbdirect_socket_init(), so that queue_work(sc->workqueue,  in smb_direct_prepare()
2298 	 * &sc->idle.immediate_work) in smb_direct_post_recv_credits()  in smb_direct_prepare()
2304 	INIT_WORK(&sc->recv_io.posted.refill_work, smb_direct_post_recv_credits);  in smb_direct_prepare()
2305 	smb_direct_post_recv_credits(&sc->recv_io.posted.refill_work);  in smb_direct_prepare()
2306 	if (unlikely(sc->first_error))  in smb_direct_prepare()
2307 		return sc->first_error;  in smb_direct_prepare()
2308 	INIT_WORK(&sc->idle.immediate_work, smb_direct_send_immediate_work);  in smb_direct_prepare()
2311 	ret = smb_direct_send_negotiate_response(sc, ret);  in smb_direct_prepare()
2316 static int smb_direct_connect(struct smbdirect_socket *sc)  in smb_direct_connect()  argument
2320 	ret = smb_direct_init_params(sc);  in smb_direct_connect()
2326 	ret = smb_direct_create_pools(sc);  in smb_direct_connect()
2332 	ret = smb_direct_create_qpair(sc);  in smb_direct_connect()
2338 	ret = smb_direct_prepare_negotiation(sc);  in smb_direct_connect()
2359 	struct smbdirect_socket *sc;  in smb_direct_handle_connect_request()  local
2376 	sc = &t->socket;  in smb_direct_handle_connect_request()
2377 	sp = &sc->parameters;  in smb_direct_handle_connect_request()
2416 			sc->rdma.legacy_iwarp = true;  in smb_direct_handle_connect_request()
2440 	ret = smb_direct_connect(sc);  in smb_direct_handle_connect_request()