30 	struct smbdirect_socket *sc = id->context;  in smbdirect_socket_rdma_event_handler()  local
43 		smbdirect_socket_status_string(sc->status),  in smbdirect_socket_rdma_event_handler()
44 		SMBDIRECT_DEBUG_ERR_PTR(sc->first_error),  in smbdirect_socket_rdma_event_handler()
45 		rdma_event_msg(sc->rdma.expected_event),  in smbdirect_socket_rdma_event_handler()
50 	sc->rdma.cm_id = NULL;  in smbdirect_socket_rdma_event_handler()
54 int smbdirect_socket_init_new(struct net *net, struct smbdirect_socket *sc)  in smbdirect_socket_init_new()  argument
59 	smbdirect_socket_init(sc);  in smbdirect_socket_init_new()
63 			    sc,  in smbdirect_socket_init_new()
79 	sc->rdma.cm_id = id;  in smbdirect_socket_init_new()
81 	INIT_WORK(&sc->disconnect_work, smbdirect_socket_cleanup_work);  in smbdirect_socket_init_new()
88 	struct smbdirect_socket *sc;  in smbdirect_socket_create_kern()  local
92 	sc = kzalloc_obj(*sc);  in smbdirect_socket_create_kern()
93 	if (!sc)  in smbdirect_socket_create_kern()
96 	ret = smbdirect_socket_init_new(net, sc);  in smbdirect_socket_create_kern()
100 	kref_init(&sc->refs.destroy);  in smbdirect_socket_create_kern()
102 	*_sc = sc;  in smbdirect_socket_create_kern()
106 	kfree(sc);  in smbdirect_socket_create_kern()
112 int smbdirect_socket_init_accepting(struct rdma_cm_id *id, struct smbdirect_socket *sc)  in smbdirect_socket_init_accepting()  argument
114 	smbdirect_socket_init(sc);  in smbdirect_socket_init_accepting()
116 	sc->rdma.cm_id = id;  in smbdirect_socket_init_accepting()
117 	sc->rdma.cm_id->context = sc;  in smbdirect_socket_init_accepting()
118 	sc->rdma.cm_id->event_handler = smbdirect_socket_rdma_event_handler;  in smbdirect_socket_init_accepting()
120 	sc->ib.dev = sc->rdma.cm_id->device;  in smbdirect_socket_init_accepting()
122 	INIT_WORK(&sc->disconnect_work, smbdirect_socket_cleanup_work);  in smbdirect_socket_init_accepting()
129 	struct smbdirect_socket *sc;  in smbdirect_socket_create_accepting()  local
133 	sc = kzalloc_obj(*sc);  in smbdirect_socket_create_accepting()
134 	if (!sc)  in smbdirect_socket_create_accepting()
137 	ret = smbdirect_socket_init_accepting(id, sc);  in smbdirect_socket_create_accepting()
141 	kref_init(&sc->refs.destroy);  in smbdirect_socket_create_accepting()
143 	*_sc = sc;  in smbdirect_socket_create_accepting()
147 	kfree(sc);  in smbdirect_socket_create_accepting()
153 int smbdirect_socket_set_initial_parameters(struct smbdirect_socket *sc,  in smbdirect_socket_set_initial_parameters()  argument
159 	WARN_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED,  in smbdirect_socket_set_initial_parameters()
161 		  smbdirect_socket_status_string(sc->status),  in smbdirect_socket_set_initial_parameters()
162 		  SMBDIRECT_DEBUG_ERR_PTR(sc->first_error));  in smbdirect_socket_set_initial_parameters()
163 	if (sc->status != SMBDIRECT_SOCKET_CREATED)  in smbdirect_socket_set_initial_parameters()
178 		rdma_restrict_node_type(sc->rdma.cm_id, RDMA_NODE_IB_CA);  in smbdirect_socket_set_initial_parameters()
180 		rdma_restrict_node_type(sc->rdma.cm_id, RDMA_NODE_RNIC);  in smbdirect_socket_set_initial_parameters()
188 	sc->parameters = *sp;  in smbdirect_socket_set_initial_parameters()
195 smbdirect_socket_get_current_parameters(struct smbdirect_socket *sc)  in smbdirect_socket_get_current_parameters()  argument
197 	return &sc->parameters;  in smbdirect_socket_get_current_parameters()
201 int smbdirect_socket_set_kernel_settings(struct smbdirect_socket *sc,  in smbdirect_socket_set_kernel_settings()  argument
208 	WARN_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED,  in smbdirect_socket_set_kernel_settings()
210 		  smbdirect_socket_status_string(sc->status),  in smbdirect_socket_set_kernel_settings()
211 		  SMBDIRECT_DEBUG_ERR_PTR(sc->first_error));  in smbdirect_socket_set_kernel_settings()
212 	if (sc->status != SMBDIRECT_SOCKET_CREATED)  in smbdirect_socket_set_kernel_settings()
215 	sc->ib.poll_ctx = poll_ctx;  in smbdirect_socket_set_kernel_settings()
217 	sc->send_io.mem.gfp_mask = gfp_mask;  in smbdirect_socket_set_kernel_settings()
218 	sc->recv_io.mem.gfp_mask = gfp_mask;  in smbdirect_socket_set_kernel_settings()
219 	sc->rw_io.mem.gfp_mask = gfp_mask;  in smbdirect_socket_set_kernel_settings()
225 void smbdirect_socket_set_logging(struct smbdirect_socket *sc,  in smbdirect_socket_set_logging()  argument
227 				  bool (*needed)(struct smbdirect_socket *sc,  in smbdirect_socket_set_logging()  argument
231 				  void (*vaprintf)(struct smbdirect_socket *sc,  in smbdirect_socket_set_logging()  argument
239 	sc->logging.private_ptr = private_ptr;  in smbdirect_socket_set_logging()
240 	sc->logging.needed = needed;  in smbdirect_socket_set_logging()
241 	sc->logging.vaprintf = vaprintf;  in smbdirect_socket_set_logging()
245 static void smbdirect_socket_wake_up_all(struct smbdirect_socket *sc)  in smbdirect_socket_wake_up_all()  argument
251 	wake_up_all(&sc->status_wait);  in smbdirect_socket_wake_up_all()
252 	wake_up_all(&sc->listen.wait_queue);  in smbdirect_socket_wake_up_all()
253 	wake_up_all(&sc->send_io.bcredits.wait_queue);  in smbdirect_socket_wake_up_all()
254 	wake_up_all(&sc->send_io.lcredits.wait_queue);  in smbdirect_socket_wake_up_all()
255 	wake_up_all(&sc->send_io.credits.wait_queue);  in smbdirect_socket_wake_up_all()
256 	wake_up_all(&sc->send_io.pending.zero_wait_queue);  in smbdirect_socket_wake_up_all()
257 	wake_up_all(&sc->recv_io.reassembly.wait_queue);  in smbdirect_socket_wake_up_all()
258 	wake_up_all(&sc->rw_io.credits.wait_queue);  in smbdirect_socket_wake_up_all()
259 	wake_up_all(&sc->mr_io.ready.wait_queue);  in smbdirect_socket_wake_up_all()
262 void __smbdirect_socket_schedule_cleanup(struct smbdirect_socket *sc,  in __smbdirect_socket_schedule_cleanup()  argument
274 	if (!sc->first_error) {  in __smbdirect_socket_schedule_cleanup()
275 		___smbdirect_log_generic(sc, func, line,  in __smbdirect_socket_schedule_cleanup()
284 			smbdirect_socket_status_string(sc->status));  in __smbdirect_socket_schedule_cleanup()
286 			sc->first_error = error;  in __smbdirect_socket_schedule_cleanup()
288 			sc->first_error = -ECONNABORTED;  in __smbdirect_socket_schedule_cleanup()
297 	disable_work(&sc->connect.work);  in __smbdirect_socket_schedule_cleanup()
298 	disable_work(&sc->recv_io.posted.refill_work);  in __smbdirect_socket_schedule_cleanup()
299 	disable_work(&sc->idle.immediate_work);  in __smbdirect_socket_schedule_cleanup()
300 	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE;  in __smbdirect_socket_schedule_cleanup()
301 	disable_delayed_work(&sc->idle.timer_work);  in __smbdirect_socket_schedule_cleanup()
309 	spin_lock_irqsave(&sc->listen.lock, flags);  in __smbdirect_socket_schedule_cleanup()
310 	list_splice_init(&sc->listen.ready, &sc->listen.pending);  in __smbdirect_socket_schedule_cleanup()
311 	list_for_each_entry_safe(psc, tsc, &sc->listen.pending, accept.list)  in __smbdirect_socket_schedule_cleanup()
312 		smbdirect_socket_schedule_cleanup(psc, sc->first_error);  in __smbdirect_socket_schedule_cleanup()
313 	spin_unlock_irqrestore(&sc->listen.lock, flags);  in __smbdirect_socket_schedule_cleanup()
315 	switch (sc->status) {  in __smbdirect_socket_schedule_cleanup()
331 		sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED;  in __smbdirect_socket_schedule_cleanup()
336 		sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED;  in __smbdirect_socket_schedule_cleanup()
341 		sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED;  in __smbdirect_socket_schedule_cleanup()
346 		sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED;  in __smbdirect_socket_schedule_cleanup()
351 		sc->status = SMBDIRECT_SOCKET_DISCONNECTED;  in __smbdirect_socket_schedule_cleanup()
355 		sc->status = SMBDIRECT_SOCKET_ERROR;  in __smbdirect_socket_schedule_cleanup()
359 	if (force_status && (was_first || *force_status > sc->status))  in __smbdirect_socket_schedule_cleanup()
360 		sc->status = *force_status;  in __smbdirect_socket_schedule_cleanup()
366 	smbdirect_socket_wake_up_all(sc);  in __smbdirect_socket_schedule_cleanup()
368 	queue_work(sc->workqueues.cleanup, &sc->disconnect_work);  in __smbdirect_socket_schedule_cleanup()
373 	struct smbdirect_socket *sc =  in smbdirect_socket_cleanup_work()  local
383 	if (!sc->first_error) {  in smbdirect_socket_cleanup_work()
384 		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR,  in smbdirect_socket_cleanup_work()
386 			smbdirect_socket_status_string(sc->status));  in smbdirect_socket_cleanup_work()
388 		sc->first_error = -ECONNABORTED;  in smbdirect_socket_cleanup_work()
396 	disable_work(&sc->disconnect_work);  in smbdirect_socket_cleanup_work()
397 	disable_work(&sc->connect.work);  in smbdirect_socket_cleanup_work()
398 	disable_work(&sc->recv_io.posted.refill_work);  in smbdirect_socket_cleanup_work()
399 	disable_work(&sc->idle.immediate_work);  in smbdirect_socket_cleanup_work()
400 	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE;  in smbdirect_socket_cleanup_work()
401 	disable_delayed_work(&sc->idle.timer_work);  in smbdirect_socket_cleanup_work()
409 	spin_lock_irqsave(&sc->listen.lock, flags);  in smbdirect_socket_cleanup_work()
410 	list_splice_init(&sc->listen.ready, &sc->listen.pending);  in smbdirect_socket_cleanup_work()
411 	list_for_each_entry_safe(psc, tsc, &sc->listen.pending, accept.list)  in smbdirect_socket_cleanup_work()
412 		smbdirect_socket_schedule_cleanup(psc, sc->first_error);  in smbdirect_socket_cleanup_work()
413 	spin_unlock_irqrestore(&sc->listen.lock, flags);  in smbdirect_socket_cleanup_work()
415 	switch (sc->status) {  in smbdirect_socket_cleanup_work()
421 		sc->status = SMBDIRECT_SOCKET_DISCONNECTING;  in smbdirect_socket_cleanup_work()
433 		rdma_lock_handler(sc->rdma.cm_id);  in smbdirect_socket_cleanup_work()
434 		rdma_disconnect(sc->rdma.cm_id);  in smbdirect_socket_cleanup_work()
435 		rdma_unlock_handler(sc->rdma.cm_id);  in smbdirect_socket_cleanup_work()
453 		sc->status = SMBDIRECT_SOCKET_DISCONNECTED;  in smbdirect_socket_cleanup_work()
466 	smbdirect_socket_wake_up_all(sc);  in smbdirect_socket_cleanup_work()
469 static void smbdirect_socket_destroy(struct smbdirect_socket *sc)  in smbdirect_socket_destroy()  argument
478 	smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,  in smbdirect_socket_destroy()
480 		smbdirect_socket_status_string(sc->status),  in smbdirect_socket_destroy()
481 		SMBDIRECT_DEBUG_ERR_PTR(sc->first_error));  in smbdirect_socket_destroy()
488 	if (sc->status == SMBDIRECT_SOCKET_DESTROYED)  in smbdirect_socket_destroy()
491 	WARN_ONCE(sc->status != SMBDIRECT_SOCKET_DISCONNECTED,  in smbdirect_socket_destroy()
493 		  smbdirect_socket_status_string(sc->status),  in smbdirect_socket_destroy()
494 		  SMBDIRECT_DEBUG_ERR_PTR(sc->first_error));  in smbdirect_socket_destroy()
499 	WARN_ONCE(sc->accept.listener,  in smbdirect_socket_destroy()
501 		  smbdirect_socket_status_string(sc->status),  in smbdirect_socket_destroy()
502 		  SMBDIRECT_DEBUG_ERR_PTR(sc->first_error));  in smbdirect_socket_destroy()
511 	smbdirect_socket_wake_up_all(sc);  in smbdirect_socket_destroy()
513 	disable_work_sync(&sc->disconnect_work);  in smbdirect_socket_destroy()
514 	disable_work_sync(&sc->connect.work);  in smbdirect_socket_destroy()
515 	disable_work_sync(&sc->recv_io.posted.refill_work);  in smbdirect_socket_destroy()
516 	disable_work_sync(&sc->idle.immediate_work);  in smbdirect_socket_destroy()
517 	disable_delayed_work_sync(&sc->idle.timer_work);  in smbdirect_socket_destroy()
519 	if (sc->rdma.cm_id)  in smbdirect_socket_destroy()
520 		rdma_lock_handler(sc->rdma.cm_id);  in smbdirect_socket_destroy()
522 	if (sc->ib.qp) {  in smbdirect_socket_destroy()
523 		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,  in smbdirect_socket_destroy()
525 		ib_drain_qp(sc->ib.qp);  in smbdirect_socket_destroy()
534 	spin_lock_irqsave(&sc->listen.lock, flags);  in smbdirect_socket_destroy()
535 	list_splice_tail_init(&sc->listen.ready, &all_list);  in smbdirect_socket_destroy()
536 	list_splice_tail_init(&sc->listen.pending, &all_list);  in smbdirect_socket_destroy()
537 	spin_unlock_irqrestore(&sc->listen.lock, flags);  in smbdirect_socket_destroy()
539 	if (sc->listen.backlog != -1) /* was a listener */  in smbdirect_socket_destroy()
540 		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,  in smbdirect_socket_destroy()
547 	if (sc->listen.backlog != -1) /* was a listener */  in smbdirect_socket_destroy()
548 		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,  in smbdirect_socket_destroy()
553 	if (sc->listen.backlog == -1) /* was not a listener */  in smbdirect_socket_destroy()
554 		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,  in smbdirect_socket_destroy()
556 	spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags);  in smbdirect_socket_destroy()
557 	list_splice_tail_init(&sc->recv_io.reassembly.list, &all_list);  in smbdirect_socket_destroy()
558 	spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags);  in smbdirect_socket_destroy()
561 	sc->recv_io.reassembly.data_length = 0;  in smbdirect_socket_destroy()
563 	if (sc->listen.backlog == -1) /* was not a listener */  in smbdirect_socket_destroy()
564 		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,  in smbdirect_socket_destroy()
566 	smbdirect_connection_destroy_mr_list(sc);  in smbdirect_socket_destroy()
568 	if (sc->listen.backlog == -1) /* was not a listener */  in smbdirect_socket_destroy()
569 		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,  in smbdirect_socket_destroy()
571 	smbdirect_connection_destroy_qp(sc);  in smbdirect_socket_destroy()
572 	if (sc->rdma.cm_id) {  in smbdirect_socket_destroy()
573 		rdma_unlock_handler(sc->rdma.cm_id);  in smbdirect_socket_destroy()
574 		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,  in smbdirect_socket_destroy()
576 		rdma_destroy_id(sc->rdma.cm_id);  in smbdirect_socket_destroy()
577 		sc->rdma.cm_id = NULL;  in smbdirect_socket_destroy()
580 	if (sc->listen.backlog == -1) /* was not a listener */  in smbdirect_socket_destroy()
581 		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,  in smbdirect_socket_destroy()
583 	smbdirect_connection_destroy_mem_pools(sc);  in smbdirect_socket_destroy()
585 	sc->status = SMBDIRECT_SOCKET_DESTROYED;  in smbdirect_socket_destroy()
587 	smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,  in smbdirect_socket_destroy()
591 void smbdirect_socket_destroy_sync(struct smbdirect_socket *sc)  in smbdirect_socket_destroy_sync()  argument
593 	smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,  in smbdirect_socket_destroy_sync()
595 		smbdirect_socket_status_string(sc->status),  in smbdirect_socket_destroy_sync()
596 		SMBDIRECT_DEBUG_ERR_PTR(sc->first_error));  in smbdirect_socket_destroy_sync()
614 	disable_work(&sc->disconnect_work);  in smbdirect_socket_destroy_sync()
616 	if (!sc->first_error)  in smbdirect_socket_destroy_sync()
622 		smbdirect_socket_schedule_cleanup_lvl(sc,  in smbdirect_socket_destroy_sync()
626 	smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,  in smbdirect_socket_destroy_sync()
628 	disable_work_sync(&sc->disconnect_work);  in smbdirect_socket_destroy_sync()
630 	smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,  in smbdirect_socket_destroy_sync()
632 	if (sc->status < SMBDIRECT_SOCKET_DISCONNECTING)  in smbdirect_socket_destroy_sync()
633 		smbdirect_socket_cleanup_work(&sc->disconnect_work);  in smbdirect_socket_destroy_sync()
634 	if (sc->status < SMBDIRECT_SOCKET_DISCONNECTED) {  in smbdirect_socket_destroy_sync()
635 		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,  in smbdirect_socket_destroy_sync()
637 		wait_event(sc->status_wait, sc->status == SMBDIRECT_SOCKET_DISCONNECTED);  in smbdirect_socket_destroy_sync()
638 		smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,  in smbdirect_socket_destroy_sync()
646 	smbdirect_socket_destroy(sc);  in smbdirect_socket_destroy_sync()
647 	smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO,  in smbdirect_socket_destroy_sync()
649 		smbdirect_socket_status_string(sc->status),  in smbdirect_socket_destroy_sync()
650 		SMBDIRECT_DEBUG_ERR_PTR(sc->first_error));  in smbdirect_socket_destroy_sync()
653 int smbdirect_socket_bind(struct smbdirect_socket *sc, struct sockaddr *addr)  in smbdirect_socket_bind()  argument
657 	if (sc->status != SMBDIRECT_SOCKET_CREATED)  in smbdirect_socket_bind()
660 	ret = rdma_bind_addr(sc->rdma.cm_id, addr);  in smbdirect_socket_bind()
668 void smbdirect_socket_shutdown(struct smbdirect_socket *sc)  in smbdirect_socket_shutdown()  argument
670 	smbdirect_socket_schedule_cleanup_lvl(sc, SMBDIRECT_LOG_INFO, -ESHUTDOWN);  in smbdirect_socket_shutdown()
676 	struct smbdirect_socket *sc =  in smbdirect_socket_release_disconnect()  local
682 	smbdirect_socket_destroy_sync(sc);  in smbdirect_socket_release_disconnect()
687 	struct smbdirect_socket *sc =  in smbdirect_socket_release_destroy()  local
695 	smbdirect_socket_destroy_sync(sc);  in smbdirect_socket_release_destroy()
696 	kfree(sc);  in smbdirect_socket_release_destroy()
699 void smbdirect_socket_release(struct smbdirect_socket *sc)  in smbdirect_socket_release()  argument
705 	WARN_ON_ONCE(kref_read(&sc->refs.disconnect) != 1);  in smbdirect_socket_release()
706 	WARN_ON(!kref_put(&sc->refs.disconnect, smbdirect_socket_release_disconnect));  in smbdirect_socket_release()
713 	kref_put(&sc->refs.destroy, smbdirect_socket_release_destroy);  in smbdirect_socket_release()
717 int smbdirect_socket_wait_for_credits(struct smbdirect_socket *sc,  in smbdirect_socket_wait_for_credits()  argument
736 					       sc->status != expected_status);  in smbdirect_socket_wait_for_credits()
738 		if (sc->status != expected_status)  in smbdirect_socket_wait_for_credits()